Merge branch 'develop' into hide-unhide-workspace

.github/helper/roulette.py

@@ -4,8 +4,10 @@ import re
 import shlex
 import subprocess
 import sys
+import time
 import urllib.request
 from functools import lru_cache
+from urllib.error import HTTPError
 
 
 @lru_cache(maxsize=None)
@@ -15,11 +17,30 @@ def fetch_pr_data(pr_number, repo, endpoint=""):
 	if endpoint:
 		api_url += f"/{endpoint}"
 
-	req = urllib.request.Request(api_url)
-	res = urllib.request.urlopen(req)
+	res = req(api_url)
 	return json.loads(res.read().decode("utf8"))
 
 
+def req(url):
+	"Simple resilient request call to handle rate limits."
+	headers = None
+	token = os.environ.get("GITHUB_TOKEN")
+	if token:
+		headers = {"authorization": f"Bearer {token}"}
+
+	retries = 0
+	while True:
+		try:
+			req = urllib.request.Request(url, headers=headers)
+			return urllib.request.urlopen(req)
+		except HTTPError as exc:
+			if exc.code == 403 and retries < 5:
+				retries += 1
+				time.sleep(retries)
+				continue
+			raise
+
+
 def get_files_list(pr_number, repo="frappe/frappe"):
 	return [change["filename"] for change in fetch_pr_data(pr_number, repo, "files")]
 

.github/workflows/create-release.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
       - name: Setup dependencies
         run: |
           npm install @semantic-release/git @semantic-release/exec --no-save

.github/workflows/patch-mariadb-tests.yml

@@ -9,6 +9,7 @@ concurrency:
   cancel-in-progress: true
 
 permissions:
+  # Do not change this as GITHUB_TOKEN is being used by roulette
   contents: read
 
 jobs:
@@ -31,6 +32,7 @@ jobs:
           TYPE: "server"
           PR_NUMBER: ${{ github.event.number }}
           REPO_NAME: ${{ github.repository }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   test:
     name: Patch

.github/workflows/server-tests.yml

@@ -12,6 +12,7 @@ concurrency:
 
 
 permissions:
+  # Do not change this as GITHUB_TOKEN is being used by roulette
   contents: read
 
 jobs:
@@ -34,6 +35,7 @@ jobs:
           TYPE: "server"
           PR_NUMBER: ${{ github.event.number }}
           REPO_NAME: ${{ github.repository }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   test:
     name: Unit Tests

.github/workflows/ui-tests.yml

@@ -11,6 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 permissions:
+  # Do not change this as GITHUB_TOKEN is being used by roulette
   contents: read
 
 jobs:
@@ -33,6 +34,7 @@ jobs:
           TYPE: "ui"
           PR_NUMBER: ${{ github.event.number }}
           REPO_NAME: ${{ github.repository }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   test:
     runs-on: ubuntu-latest

frappe/custom/doctype/customize_form/customize_form.js

@@ -111,54 +111,59 @@ frappe.ui.form.on("Customize Form", {
 		frm.page.clear_icons();
 
 		if (frm.doc.doc_type) {
-			frm.page.set_title(__("Customize Form - {0}", [frm.doc.doc_type]));
-			frappe.customize_form.set_primary_action(frm);
+			frappe.model.with_doctype(frm.doc.doc_type).then(() => {
+				frm.page.set_title(__("Customize Form - {0}", [frm.doc.doc_type]));
+				frappe.customize_form.set_primary_action(frm);
 
-			if (!frm.is_new()) {
-				frm.add_custom_button(__("Try new form builder", [__(frm.doc.doc_type)]), () => {
-					frappe.set_route("form-builder", frm.doc.doc_type, "customize");
-				});
-			}
+				if (!frm.is_new()) {
+					frm.add_custom_button(
+						__("Try new form builder", [__(frm.doc.doc_type)]),
+						() => {
+							frappe.set_route("form-builder", frm.doc.doc_type, "customize");
+						}
+					);
+				}
 
-			frm.add_custom_button(
-				__("Go to {0} List", [__(frm.doc.doc_type)]),
-				function () {
-					frappe.set_route("List", frm.doc.doc_type);
-				},
-				__("Actions")
-			);
+				frm.add_custom_button(
+					__("Go to {0} List", [__(frm.doc.doc_type)]),
+					function () {
+						frappe.set_route("List", frm.doc.doc_type);
+					},
+					__("Actions")
+				);
 
-			frm.add_custom_button(
-				__("Reload"),
-				function () {
-					frm.script_manager.trigger("doc_type");
-				},
-				__("Actions")
-			);
+				frm.add_custom_button(
+					__("Reload"),
+					function () {
+						frm.script_manager.trigger("doc_type");
+					},
+					__("Actions")
+				);
 
-			frm.add_custom_button(
-				__("Reset to defaults"),
-				function () {
-					frappe.customize_form.confirm(__("Remove all customizations?"), frm);
-				},
-				__("Actions")
-			);
+				frm.add_custom_button(
+					__("Reset to defaults"),
+					function () {
+						frappe.customize_form.confirm(__("Remove all customizations?"), frm);
+					},
+					__("Actions")
+				);
 
-			frm.add_custom_button(
-				__("Set Permissions"),
-				function () {
-					frappe.set_route("permission-manager", frm.doc.doc_type);
-				},
-				__("Actions")
-			);
+				frm.add_custom_button(
+					__("Set Permissions"),
+					function () {
+						frappe.set_route("permission-manager", frm.doc.doc_type);
+					},
+					__("Actions")
+				);
 
-			const is_autoname_autoincrement = frm.doc.autoname === "autoincrement";
-			frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement);
-			frm.set_df_property("autoname", "read_only", is_autoname_autoincrement);
-			frm.toggle_display(
-				["queue_in_background"],
-				frappe.get_meta(frm.doc.doc_type).is_submittable || 0
-			);
+				const is_autoname_autoincrement = frm.doc.autoname === "autoincrement";
+				frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement);
+				frm.set_df_property("autoname", "read_only", is_autoname_autoincrement);
+				frm.toggle_display(
+					["queue_in_background"],
+					frappe.get_meta(frm.doc.doc_type).is_submittable || 0
+				);
+			});
 		}
 
 		frm.events.setup_export(frm);

frappe/desk/page/user_profile/user_profile.py

@@ -1,6 +1,8 @@
 from datetime import datetime
 
 import frappe
+from frappe.query_builder import Interval, Order
+from frappe.query_builder.functions import Date, Sum, UnixTimestamp
 from frappe.utils import getdate
 
 
@@ -11,21 +13,18 @@ def get_energy_points_heatmap_data(user, date):
 	except Exception:
 		date = getdate()
 
+	eps_log = frappe.qb.DocType("Energy Point Log")
+
 	return dict(
-		frappe.db.sql(
-			"""select unix_timestamp(date(creation)), sum(points)
-		from `tabEnergy Point Log`
-		where
-			date(creation) > subdate('{date}', interval 1 year) and
-			date(creation) < subdate('{date}', interval -1 year) and
-			user = %s and
-			type != 'Review'
-		group by date(creation)
-		order by creation asc""".format(
-				date=date
-			),
-			user,
-		)
+		frappe.qb.from_(eps_log)
+		.select(UnixTimestamp(Date(eps_log.creation)), Sum(eps_log.points))
+		.where(eps_log.user == user)
+		.where(eps_log["type"] != "Review")
+		.where(Date(eps_log.creation) > Date(date) - Interval(years=1))
+		.where(Date(eps_log.creation) < Date(date) + Interval(years=1))
+		.groupby(Date(eps_log.creation))
+		.orderby(Date(eps_log.creation), order=Order.asc)
+		.run()
 	)
 
 
@@ -51,7 +50,7 @@ def get_user_rank(user):
 	month_start = datetime.today().replace(day=1)
 	monthly_rank = frappe.get_all(
 		"Energy Point Log",
-		group_by="user",
+		group_by="`tabEnergy Point Log`.`user`",
 		filters={"creation": [">", month_start], "type": ["!=", "Review"]},
 		fields=["user", "sum(points)"],
 		order_by="sum(points) desc",
@@ -60,7 +59,7 @@ def get_user_rank(user):
 
 	all_time_rank = frappe.get_all(
 		"Energy Point Log",
-		group_by="user",
+		group_by="`tabEnergy Point Log`.`user`",
 		filters={"type": ["!=", "Review"]},
 		fields=["user", "sum(points)"],
 		order_by="sum(points) desc",

frappe/desk/reportview.py

@@ -20,7 +20,7 @@ from frappe.utils import add_user_info, format_duration
 @frappe.read_only()
 def get():
 	args = get_form_params()
-	# If virtual doctype get data from controller het_list method
+	# If virtual doctype, get data from controller get_list method
 	if is_virtual_doctype(args.doctype):
 		controller = get_controller(args.doctype)
 		data = compress(controller.get_list(args))

frappe/hooks.py

@@ -392,4 +392,5 @@ ignore_links_on_delete = [
 	"Email Queue",
 	"Document Share Key",
 	"Integration Request",
+	"Unhandled Email",
 ]

frappe/permissions.py

@@ -27,28 +27,12 @@ rights = (
 )
 
 
-def check_admin_or_system_manager(user=None):
-	from frappe.utils.commands import warn
-
-	warn(
-		"The function check_admin_or_system_manager will be deprecated in version 15."
-		'Please use frappe.only_for("System Manager") instead.',
-		category=PendingDeprecationWarning,
-	)
-
-	if not user:
-		user = frappe.session.user
-
-	if ("System Manager" not in frappe.get_roles(user)) and (user != "Administrator"):
-		frappe.throw(_("Not permitted"), frappe.PermissionError)
-
-
 def print_has_permission_check_logs(func):
 	def inner(*args, **kwargs):
 		frappe.flags["has_permission_check_logs"] = []
 		result = func(*args, **kwargs)
 		self_perm_check = True if not kwargs.get("user") else kwargs.get("user") == frappe.session.user
-		raise_exception = False if kwargs.get("raise_exception") is False else True
+		raise_exception = kwargs.get("raise_exception", True)
 
 		# print only if access denied
 		# and if user is checking his own permission

frappe/query_builder/functions.py

@@ -74,6 +74,22 @@ DateFormat = ImportMapper(
 )
 
 
+class _PostgresUnixTimestamp(Extract):
+	# Note: this is just a special case of "Extract" function with "epoch" hardcoded.
+	# Check super definition to see how it works.
+	def __init__(self, field, alias=None):
+		super().__init__("epoch", field=field, alias=alias)
+		self.field = field
+
+
+UnixTimestamp = ImportMapper(
+	{
+		db_type_is.MARIADB: CustomFunction("unix_timestamp", ["date"]),
+		db_type_is.POSTGRES: _PostgresUnixTimestamp,
+	}
+)
+
+
 class Cast_(Function):
 	def __init__(self, value, as_type, alias=None):
 		if frappe.db.db_type == "mariadb" and (

frappe/social/doctype/energy_point_log/test_energy_point_log.py

@@ -2,6 +2,7 @@
 # License: MIT. See LICENSE
 import frappe
 from frappe.desk.form.assign_to import add as assign_to
+from frappe.desk.page.user_profile.user_profile import get_energy_points_heatmap_data
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils.testutils import add_custom_field, clear_custom_fields
 
@@ -234,6 +235,10 @@ class TestEnergyPointLog(FrappeTestCase):
 
 		self.assertEqual(test2_user_after_points, test2_user_before_points + rule.points)
 
+	def test_eps_heatmap_query(self):
+		# Just asserts that query works, not correctness.
+		self.assertIsInstance(get_energy_points_heatmap_data(user="test@example.com", date=None), dict)
+
 	def test_points_on_field_value_change(self):
 		rule = create_energy_point_rule_for_todo(
 			for_doc_event="Value Change", field_to_check="description"

frappe/tests/test_query_builder.py

@@ -6,7 +6,15 @@ import frappe
 from frappe.query_builder import Case
 from frappe.query_builder.builder import Function
 from frappe.query_builder.custom import ConstantColumn
-from frappe.query_builder.functions import Cast_, Coalesce, CombineDatetime, GroupConcat, Match
+from frappe.query_builder.functions import (
+	Cast_,
+	Coalesce,
+	CombineDatetime,
+	Date,
+	GroupConcat,
+	Match,
+	UnixTimestamp,
+)
 from frappe.query_builder.utils import db_type_is
 from frappe.tests.utils import FrappeTestCase
 
@@ -75,6 +83,47 @@ class TestCustomFunctionsMariaDB(FrappeTestCase):
 			str(select_query).lower(),
 		)
 
+	def test_unix_ts_mariadb(self):
+		# Simple Query
+		note = frappe.qb.DocType("Note")
+		self.assertEqual(
+			"unix_timestamp(posting_date)",
+			UnixTimestamp(note.posting_date).get_sql(),
+		)
+
+		# Complex multi table query
+		todo = frappe.qb.DocType("ToDo")
+		select_query = (
+			frappe.qb.from_(note)
+			.join(todo)
+			.on(todo.refernce_name == note.name)
+			.select(UnixTimestamp(note.posting_date))
+		)
+		self.assertIn("select unix_timestamp(`tabnote`.`posting_date`)", str(select_query).lower())
+
+		# Order by
+		select_query = select_query.orderby(UnixTimestamp(note.posting_date))
+		self.assertIn(
+			"order by unix_timestamp(`tabnote`.`posting_date`)",
+			str(select_query).lower(),
+		)
+
+		# Function comparison
+		select_query = select_query.where(
+			UnixTimestamp(note.posting_date) >= UnixTimestamp("2021-01-01")
+		)
+		self.assertIn(
+			"unix_timestamp(`tabnote`.`posting_date`)>=unix_timestamp('2021-01-01')",
+			str(select_query).lower(),
+		)
+
+		# aliasing
+		select_query = select_query.select(UnixTimestamp(note.posting_date, alias="unix_ts"))
+		self.assertIn(
+			"unix_timestamp(`tabnote`.`posting_date`) `unix_ts`",
+			str(select_query).lower(),
+		)
+
 	def test_time(self):
 		note = frappe.qb.DocType("Note")
 		self.assertEqual(
@@ -162,6 +211,47 @@ class TestCustomFunctionsPostgres(FrappeTestCase):
 			'"tabnote"."posting_date"+"tabnote"."posting_time" "timestamp"', str(select_query).lower()
 		)
 
+	def test_unix_ts_postgres(self):
+		# Simple Query
+		note = frappe.qb.DocType("Note")
+		self.assertEqual(
+			"extract(epoch from posting_date)",
+			UnixTimestamp(note.posting_date).get_sql().lower(),
+		)
+
+		# Complex multi table query
+		todo = frappe.qb.DocType("ToDo")
+		select_query = (
+			frappe.qb.from_(note)
+			.join(todo)
+			.on(todo.refernce_name == note.name)
+			.select(UnixTimestamp(note.posting_date))
+		)
+		self.assertIn('extract(epoch from "tabnote"."posting_date")', str(select_query).lower())
+
+		# Order by
+		select_query = select_query.orderby(UnixTimestamp(note.posting_date))
+		self.assertIn(
+			'order by extract(epoch from "tabnote"."posting_date")',
+			str(select_query).lower(),
+		)
+
+		# Function comparison
+		select_query = select_query.where(
+			UnixTimestamp(note.posting_date) >= UnixTimestamp(Date("2021-01-01"))
+		)
+		self.assertIn(
+			'extract(epoch from "tabnote"."posting_date")>=extract(epoch from date(\'2021-01-01\'))',
+			str(select_query).lower(),
+		)
+
+		# aliasing
+		select_query = select_query.select(UnixTimestamp(note.posting_date, alias="unix_ts"))
+		self.assertIn(
+			'extract(epoch from "tabnote"."posting_date") "unix_ts"',
+			str(select_query).lower(),
+		)
+
 	def test_time(self):
 		note = frappe.qb.DocType("Note")