Merge branch 'develop' into csv-params

.flake8

@@ -1,37 +1,74 @@
 [flake8]
 ignore =
+    B001,
+    B007,
+    B009,
+    B010,
+    B950,
+    E101,
+    E111,
+    E114,
+    E116,
+    E117,
     E121,
+    E122,
+    E123,
+    E124,
+    E125,
     E126,
     E127,
     E128,
+    E131,
+    E201,
+    E202,
     E203,
+    E211,
+    E221,
+    E222,
+    E223,
+    E224,
     E225,
     E226,
+    E228,
     E231,
     E241,
+    E242,
     E251,
     E261,
+    E262,
     E265,
+    E266,
+    E271,
+    E272,
+    E273,
+    E274,
+    E301,
     E302,
     E303,
     E305,
+    E306,
     E402,
     E501,
+    E502,
+    E701,
+    E702,
+    E703,
     E741,
+    F401,
+    F403,
+    F405,
+    W191,
     W291,
     W292,
     W293,
     W391,
     W503,
     W504,
-    F403,
-    B007,
-    B950,
-    W191,
-    E124, # closing bracket, irritating while writing QB code
-    E131, # continuation line unaligned for hanging indent
-    E123, # closing bracket does not match indentation of opening bracket's line
-    E101, # ensured by use of black
+    E711,
+    E129,
+    F841,
+    E713,
+    E712,
 
 max-line-length = 200
-exclude=.github/helper/semgrep_rules
+exclude=,test_*.py

.github/helper/flake8.conf

@@ -1,75 +0,0 @@
-[flake8]
-ignore =
-    B001,
-    B007,
-    B009,
-    B010,
-    B950,
-    E101,
-    E111,
-    E114,
-    E116,
-    E117,
-    E121,
-    E122,
-    E123,
-    E124,
-    E125,
-    E126,
-    E127,
-    E128,
-    E131,
-    E201,
-    E202,
-    E203,
-    E211,
-    E221,
-    E222,
-    E223,
-    E224,
-    E225,
-    E226,
-    E228,
-    E231,
-    E241,
-    E242,
-    E251,
-    E261,
-    E262,
-    E265,
-    E266,
-    E271,
-    E272,
-    E273,
-    E274,
-    E301,
-    E302,
-    E303,
-    E305,
-    E306,
-    E402,
-    E501,
-    E502,
-    E701,
-    E702,
-    E703,
-    E741,
-    F401,
-    F403,
-    F405,
-    W191,
-    W291,
-    W292,
-    W293,
-    W391,
-    W503,
-    W504,
-    E711,
-    E129,
-    F841,
-    E713,
-    E712,
-
-
-max-line-length = 200
-exclude=.github/helper/semgrep_rules,test_*.py

.github/workflows/linters.yml

@@ -42,7 +42,7 @@ jobs:
       - name: 'Setup Environment'
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - uses: actions/checkout@v3
 
       - name: Validate Docs

.github/workflows/publish-assets-develop.yml

@@ -19,7 +19,7 @@ jobs:
           node-version: 16
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/server-mariadb-tests.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Check for valid Python & Merge Conflicts
         run: |

.github/workflows/server-postgres-tests.yml

@@ -66,7 +66,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Check for valid Python & Merge Conflicts
         run: |

.github/workflows/ui-tests.yml

@@ -64,7 +64,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Check for valid Python & Merge Conflicts
         run: |
@@ -121,7 +121,6 @@ jobs:
           DB: mariadb
 
       - name: Verify yarn.lock
-        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           cd ~/frappe-bench/apps/frappe
           yarn install --immutable --immutable-cache --check-cache

.mergify.yml

@@ -71,15 +71,6 @@ pull_request_rules:
         assignees:
           - "{{ author }}"
 
-  - name: backport to develop
-    conditions:
-      - label="backport develop"
-    actions:
-      backport:
-        branches:
-          - develop
-        assignees:
-          - "{{ author }}"
 
   - name: backport to version-13-pre-release
     conditions:

.pre-commit-config.yaml

@@ -59,7 +59,6 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies: ['flake8-bugbear',]
-        args: ['--config', '.github/helper/flake8.conf']
 
 ci:
     autoupdate_schedule: weekly

.stylelintrc

@@ -1,9 +0,0 @@
-{
-	"extends": ["stylelint-config-recommended"],
-	"plugins": ["stylelint-scss"],
-	"rules": {
-		"at-rule-no-unknown": null,
-		"scss/at-rule-no-unknown": true,
-		"no-descending-specificity": null
-	}
-}

bandit.yml

@@ -1 +0,0 @@
-skips: ['E0203', 'B605', 'B404', 'B603', 'B607']

cypress/integration/dashboard.js

@@ -0,0 +1,50 @@
+describe("Dashboard view", { scrollBehavior: false }, () => {
+	before(() => {
+		cy.login();
+		cy.visit("/app");
+	});
+
+	it("should load", () => {
+		const chart = "TODO-YEARLY-TRENDS";
+		const dashboard = "TODO-TEST-DASHBOARD"; // check slash in name intentionally.
+
+		cy.insert_doc(
+			"Dashboard Chart",
+			{
+				is_standard: 0,
+				chart_name: chart,
+				chart_type: "Count",
+				document_type: "ToDo",
+				parent_document_type: "",
+				based_on: "creation",
+				group_by_type: "Count",
+				timespan: "Last Year",
+				time_interval: "Yearly",
+				timeseries: 1,
+				type: "Line",
+				filters_json: "[]",
+			},
+			true
+		);
+
+		cy.insert_doc(
+			"Dashboard",
+			{
+				name: dashboard,
+				dashboard_name: dashboard,
+				is_standard: 0,
+				charts: [
+					{
+						chart: chart,
+					},
+				],
+			},
+			true
+		);
+
+		cy.visit(`/app/dashboard-view/${dashboard}`);
+
+		// expect chart to be loaded
+		cy.findByText(chart).should("be.visible");
+	});
+});

frappe/__init__.py

@@ -1018,19 +1018,15 @@ def get_precision(
 	return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency)
 
 
-def generate_hash(txt: str | None = None, length: int | None = None) -> str:
-	"""Generates random hash for given text + current timestamp + random string."""
-	import hashlib
-	import time
+def generate_hash(txt: str | None = None, length: int = 56) -> str:
+	"""Generate random hash using best available randomness source."""
+	import math
+	import secrets
 
-	from .utils import random_string
+	if not length:
+		length = 56
 
-	digest = hashlib.sha224(
-		((txt or "") + repr(time.time()) + repr(random_string(8))).encode()
-	).hexdigest()
-	if length:
-		digest = digest[:length]
-	return digest
+	return secrets.token_hex(math.ceil(length / 2))[:length]
 
 
 def reset_metadata_version():

frappe/app.py

@@ -12,13 +12,13 @@ from werkzeug.wrappers import Request, Response
 
 import frappe
 import frappe.api
-import frappe.auth
 import frappe.handler
 import frappe.monitor
 import frappe.rate_limiter
 import frappe.recorder
 import frappe.utils.response
 from frappe import _
+from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest
 from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request
 from frappe.middlewares import StaticDataMiddleware
 from frappe.utils import get_site_name, sanitize_html
@@ -29,8 +29,6 @@ local_manager = LocalManager(frappe.local)
 
 _site = None
 _sites_path = os.environ.get("SITES_PATH", ".")
-SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS")
-UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH")
 
 
 @local_manager.middleware
@@ -88,7 +86,8 @@ def application(request: Request):
 
 		log_request(request, response)
 		process_response(response)
-		frappe.destroy()
+		if frappe.db:
+			frappe.db.close()
 
 	return response
 
@@ -118,7 +117,7 @@ def init_request(request):
 	make_form_dict(request)
 
 	if request.method != "OPTIONS":
-		frappe.local.http_request = frappe.auth.HTTPRequest()
+		frappe.local.http_request = HTTPRequest()
 
 
 def setup_read_only_mode():

frappe/auth.py

@@ -20,6 +20,9 @@ from frappe.utils import cint, date_diff, datetime, get_datetime, today
 from frappe.utils.password import check_password
 from frappe.website.utils import get_home_page
 
+SAFE_HTTP_METHODS = frozenset(("GET", "HEAD", "OPTIONS"))
+UNSAFE_HTTP_METHODS = frozenset(("POST", "PUT", "DELETE", "PATCH"))
+
 
 class HTTPRequest:
 	def __init__(self):
@@ -67,25 +70,21 @@ class HTTPRequest:
 		frappe.local.login_manager = LoginManager()
 
 	def validate_csrf_token(self):
-		if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"):
-			if not frappe.local.session:
-				return
-			if (
-				not frappe.local.session.data.csrf_token
-				or frappe.local.session.data.device == "mobile"
-				or frappe.conf.get("ignore_csrf", None)
-			):
-				# not via boot
-				return
+		if (
+			not frappe.request
+			or frappe.request.method not in UNSAFE_HTTP_METHODS
+			or frappe.conf.ignore_csrf
+			or not frappe.session
+			or not (saved_token := frappe.session.data.csrf_token)
+			or (
+				(frappe.get_request_header("X-Frappe-CSRF-Token") or frappe.form_dict.pop("csrf_token", None))
+				== saved_token
+			)
+		):
+			return
 
-			csrf_token = frappe.get_request_header("X-Frappe-CSRF-Token")
-			if not csrf_token and "csrf_token" in frappe.local.form_dict:
-				csrf_token = frappe.local.form_dict.csrf_token
-				del frappe.local.form_dict["csrf_token"]
-
-			if frappe.local.session.data.csrf_token != csrf_token:
-				frappe.local.flags.disable_traceback = True
-				frappe.throw(_("Invalid Request"), frappe.CSRFTokenError)
+		frappe.flags.disable_traceback = True
+		frappe.throw(_("Invalid Request"), frappe.CSRFTokenError)
 
 	def set_lang(self):
 		frappe.local.lang = get_language()
@@ -354,10 +353,6 @@ class CookieManager:
 		if not secure and hasattr(frappe.local, "request"):
 			secure = frappe.local.request.scheme == "https"
 
-		# Cordova does not work with Lax
-		if frappe.local.session.data.device == "mobile":
-			samesite = None
-
 		self.cookies[key] = {
 			"value": value,
 			"expires": expires,

frappe/cache_manager.py

@@ -8,11 +8,19 @@ from frappe.desk.notifications import clear_notifications, delete_notification_c
 
 common_default_keys = ["__default", "__global"]
 
-doctype_map_keys = (
-	"energy_point_rule_map",
-	"assignment_rule_map",
-	"milestone_tracker_map",
-)
+doctypes_for_mapping = {
+	"Energy Point Rule",
+	"Assignment Rule",
+	"Milestone Tracker",
+	"Document Naming Rule",
+}
+
+
+def get_doctype_map_key(doctype):
+	return frappe.scrub(doctype) + "_map"
+
+
+doctype_map_keys = tuple(map(get_doctype_map_key, doctypes_for_mapping))
 
 bench_cache_keys = ("assets_json",)
 
@@ -66,7 +74,7 @@ doctype_cache_keys = (
 	"notifications",
 	"workflow",
 	"data_import_column_header_map",
-) + doctype_map_keys
+)
 
 
 def clear_user_cache(user=None):
@@ -161,23 +169,11 @@ def clear_controller_cache(doctype=None):
 
 
 def get_doctype_map(doctype, name, filters=None, order_by=None):
-	cache = frappe.cache()
-	cache_key = frappe.scrub(doctype) + "_map"
-	doctype_map = cache.hget(cache_key, name)
-
-	if doctype_map is not None:
-		# cached, return
-		items = json.loads(doctype_map)
-	else:
-		# non cached, build cache
-		try:
-			items = frappe.get_all(doctype, filters=filters, order_by=order_by)
-			cache.hset(cache_key, name, json.dumps(items))
-		except frappe.db.TableMissingError:
-			# executed from inside patch, ignore
-			items = []
-
-	return items
+	return frappe.cache().hget(
+		get_doctype_map_key(doctype),
+		name,
+		lambda: frappe.get_all(doctype, filters=filters, order_by=order_by, ignore_ddl=True),
+	)
 
 
 def clear_doctype_map(doctype, name):

frappe/client.py

@@ -333,11 +333,6 @@ def get_js(items):
 		with open(contentpath) as srcfile:
 			code = frappe.utils.cstr(srcfile.read())
 
-		if frappe.local.lang != "en":
-			messages = frappe.get_lang_dict("jsfile", contentpath)
-			messages = json.dumps(messages)
-			code += f"\n\n$.extend(frappe._messages, {messages})"
-
 		out.append(code)
 
 	return out

frappe/core/doctype/data_export/exporter.py

@@ -432,7 +432,7 @@ class DataExporter:
 				row[_column_start_end.start + i + 1] = value
 
 	def build_response_as_excel(self):
-		filename = frappe.generate_hash("", 10)
+		filename = frappe.generate_hash(length=10)
 		with open(filename, "wb") as f:
 			f.write(cstr(self.writer.getvalue()).encode("utf-8"))
 		f = open(filename)

frappe/core/doctype/data_import/test_importer.py

@@ -97,7 +97,7 @@ class TestImporter(FrappeTestCase):
 	def test_data_import_update(self):
 		existing_doc = frappe.get_doc(
 			doctype=doctype_name,
-			title=frappe.generate_hash(doctype_name, 8),
+			title=frappe.generate_hash(length=8),
 			table_field_1=[{"child_title": "child title to update"}],
 		)
 		existing_doc.save()

frappe/core/doctype/doctype/boilerplate/controller.js

@@ -1,8 +1,8 @@
 // Copyright (c) {year}, {app_publisher} and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('{doctype}', {{
-	// refresh: function(frm) {{
+// frappe.ui.form.on("{doctype}", {{
+// 	refresh(frm) {{
 
-	// }}
-}});
+// 	}},
+// }});

frappe/core/doctype/doctype/boilerplate/controller_list.js

@@ -1,5 +1,5 @@
 /* eslint-disable */
-frappe.listview_settings['{doctype}'] = {{
-	// add_fields: ["status"],
-	// filters:[["status","=", "Open"]]
-}};
+// frappe.listview_settings["{doctype}"] = {{
+// 	add_fields: ["status"],
+// 	filters: [["status","=", "Open"]],
+// }};

frappe/core/doctype/doctype/doctype.json

@@ -24,6 +24,7 @@
   "custom",
   "beta",
   "is_virtual",
+  "queue_in_background",
   "fields_section_break",
   "fields",
   "sb1",
@@ -600,6 +601,13 @@
    "fieldtype": "Check",
    "label": "Make Attachments Public by Default"
   },
+  {
+   "default": "0",
+   "depends_on": "eval: doc.is_submittable",
+   "fieldname": "queue_in_background",
+   "fieldtype": "Check",
+   "label": "Queue in Background"
+  },
   {
    "fieldname": "default_view",
    "fieldtype": "Select",

frappe/core/doctype/doctype/doctype.py

@@ -329,7 +329,7 @@ class DocType(Document):
 			"DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name)
 		)
 		for p in parent_list:
-			frappe.db.update("DocType", p.parent, {}, for_update=False)
+			frappe.db.set_value("DocType", p.parent, {}, for_update=False)
 
 	def scrub_field_names(self):
 		"""Sluggify fieldnames if not set from Label."""

frappe/core/doctype/doctype/test_doctype.py

@@ -670,6 +670,9 @@ class TestDocType(FrappeTestCase):
 
 		self.assertEqual(test_json.test_json_field["hello"], "world")
 
+	def test_no_delete_doc(self):
+		self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address")
+
 	@patch.dict(frappe.conf, {"developer_mode": 1})
 	def test_custom_field_deletion(self):
 		"""Custom child tables whose doctype doesn't exist should be auto deleted."""

frappe/core/doctype/document_naming_rule/document_naming_rule.py

@@ -12,6 +12,15 @@ class DocumentNamingRule(Document):
 	def validate(self):
 		self.validate_fields_in_conditions()
 
+	def clear_doctype_map(self):
+		frappe.cache_manager.clear_doctype_map(self.doctype, self.document_type)
+
+	def on_update(self):
+		self.clear_doctype_map()
+
+	def on_trash(self):
+		self.clear_doctype_map()
+
 	def validate_fields_in_conditions(self):
 		if self.has_value_changed("document_type"):
 			docfields = [x.fieldname for x in frappe.get_meta(self.document_type).fields]

frappe/core/doctype/log_settings/log_settings.py

@@ -17,6 +17,7 @@ DEFAULT_LOGTYPES_RETENTION = {
 	"Error Snapshot": 30,
 	"Scheduled Job Log": 90,
 	"Route History": 90,
+	"Submission Queue": 30,
 }
 
 
@@ -68,6 +69,9 @@ class LogSettings(Document):
 		added_logtypes = set()
 		for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items():
 			if logtype not in existing_logtypes and _supports_log_clearing(logtype):
+				if not frappe.db.exists("DocType", logtype):
+					continue
+
 				self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)})
 				added_logtypes.add(logtype)
 
@@ -151,6 +155,7 @@ LOG_DOCTYPES = [
 	"Email Queue Recipient",
 	"Error Snapshot",
 	"Error Log",
+	"Submission Queue",
 ]
 
 

frappe/core/doctype/rq_job/rq_job.py

@@ -5,10 +5,12 @@ import functools
 import re
 
 from rq.command import send_stop_job_command
+from rq.exceptions import InvalidJobOperation
 from rq.job import Job
 from rq.queue import Queue
 
 import frappe
+from frappe import _
 from frappe.model.document import Document
 from frappe.utils import (
 	cint,
@@ -93,7 +95,10 @@ class RQJob(Document):
 
 	@check_permissions
 	def stop_job(self):
-		send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id)
+		try:
+			send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id)
+		except InvalidJobOperation:
+			frappe.msgprint(_("Job is not running."), title=_("Invalid Operation"))
 
 	@staticmethod
 	def get_count(args) -> int:

frappe/core/doctype/rq_job/test_rq_job.py

@@ -19,12 +19,11 @@ class TestRQJob(FrappeTestCase):
 
 	@timeout(seconds=20)
 	def check_status(self, job: Job, status, wait=True):
-		if wait:
-			while True:
-				if job.is_queued or job.is_started:
-					time.sleep(0.2)
-				else:
-					break
+		while wait:
+			if not (job.is_queued or job.is_started):
+				break
+			time.sleep(0.2)
+
 		self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status)
 
 	def test_serialization(self):
@@ -69,7 +68,7 @@ class TestRQJob(FrappeTestCase):
 		self.assertGreaterEqual(len(non_failed_jobs), 1)
 
 		# Create a slow job and check if it's stuck in "Started"
-		job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=1000)
+		job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=10)
 		time.sleep(3)
 		self.check_status(job, "started", wait=False)
 		stop_job(job_id=job.id)
@@ -84,8 +83,8 @@ class TestRQJob(FrappeTestCase):
 
 	def test_is_enqueued(self):
 
+		dummy_job = frappe.enqueue(self.BG_JOB, sleep=10, queue="short")
 		job_name = "uniq_test_job"
-		dummy_job = frappe.enqueue(self.BG_JOB, sleep=100, queue="short")
 		actual_job = frappe.enqueue(self.BG_JOB, job_name=job_name, queue="short")
 
 		self.assertTrue(is_job_queued(job_name))

frappe/core/doctype/rq_worker/rq_worker.json

@@ -20,7 +20,8 @@
   "column_break_12",
   "birth_date",
   "last_heartbeat",
-  "total_working_time"
+  "total_working_time",
+  "utilization_percent"
  ],
  "fields": [
   {
@@ -59,7 +60,6 @@
   {
    "fieldname": "successful_job_count",
    "fieldtype": "Int",
-   "in_list_view": 1,
    "label": "Successful Job Count"
   },
   {
@@ -102,12 +102,18 @@
   {
    "fieldname": "column_break_12",
    "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "utilization_percent",
+   "fieldtype": "Percent",
+   "in_list_view": 1,
+   "label": "Utilization %"
   }
  ],
  "in_create": 1,
  "is_virtual": 1,
  "links": [],
- "modified": "2022-09-11 05:02:53.981705",
+ "modified": "2022-11-14 15:35:32.786012",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "RQ Worker",

frappe/core/doctype/rq_worker/rq_worker.py

@@ -1,6 +1,9 @@
 # Copyright (c) 2022, Frappe Technologies and contributors
 # For license information, please see license.txt
 
+import datetime
+from contextlib import suppress
+
 from rq import Worker
 
 import frappe
@@ -66,4 +69,11 @@ def serialize_worker(worker: Worker) -> frappe._dict:
 		_comment_count=0,
 		modified=convert_utc_to_user_timezone(worker.last_heartbeat),
 		creation=convert_utc_to_user_timezone(worker.birth_date),
+		utilization_percent=compute_utilization(worker),
 	)
+
+
+def compute_utilization(worker: Worker) -> float:
+	with suppress(Exception):
+		total_time = (datetime.datetime.utcnow() - worker.birth_date).total_seconds()
+		return worker.total_working_time / total_time * 100

frappe/core/doctype/server_script/test_server_script.py

@@ -211,3 +211,25 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
 """
 		script.save()
 		script.execute_method()
+
+	def test_scripts_all_the_way_down(self):
+		# why not
+		script = frappe.get_doc(
+			doctype="Server Script",
+			name="test_nested_scripts_1",
+			script_type="API",
+			api_method="test_nested_scripts_1",
+			script=f"""log("nothing")""",
+		)
+		script.insert()
+		script.execute_method()
+
+		script = frappe.get_doc(
+			doctype="Server Script",
+			name="test_nested_scripts_2",
+			script_type="API",
+			api_method="test_nested_scripts_2",
+			script=f"""frappe.call("test_nested_scripts_1")""",
+		)
+		script.insert()
+		script.execute_method()

frappe/core/doctype/submission_queue/submission_queue.js

@@ -0,0 +1,14 @@
+// Copyright (c) 2022, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("Submission Queue", {
+	refresh: function (frm) {
+		if (frm.doc.status === "Queued" && frm.doc.job_id) {
+			frm.add_custom_button(__("Unlock Reference Document"), () => {
+				frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => {
+					frm.call("unlock_doc");
+				});
+			});
+		}
+	},
+});

frappe/core/doctype/submission_queue/submission_queue.json

@@ -0,0 +1,123 @@
+{
+ "actions": [],
+ "autoname": "hash",
+ "creation": "2022-10-04 00:41:00.028163",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "status",
+  "created_at",
+  "enqueued_by",
+  "job_id",
+  "column_break_5",
+  "ended_at",
+  "ref_doctype",
+  "ref_docname",
+  "section_break_8",
+  "exception"
+ ],
+ "fields": [
+  {
+   "fieldname": "job_id",
+   "fieldtype": "Data",
+   "label": "Job Id",
+   "read_only": 1
+  },
+  {
+   "fieldname": "ref_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Reference DocType",
+   "options": "DocType",
+   "read_only": 1
+  },
+  {
+   "fieldname": "ref_docname",
+   "fieldtype": "Dynamic Link",
+   "in_list_view": 1,
+   "label": "Reference Docname",
+   "options": "ref_doctype",
+   "read_only": 1,
+   "search_index": 1
+  },
+  {
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "hidden": 1,
+   "in_list_view": 1,
+   "label": "Status",
+   "options": "Queued\nFinished\nFailed",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_5",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_8",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "enqueued_by",
+   "fieldtype": "Data",
+   "is_virtual": 1,
+   "label": "Enqueued By",
+   "read_only": 1
+  },
+  {
+   "fieldname": "ended_at",
+   "fieldtype": "Datetime",
+   "label": "Ended At",
+   "read_only": 1
+  },
+  {
+   "fieldname": "created_at",
+   "fieldtype": "Datetime",
+   "is_virtual": 1,
+   "label": "Created At",
+   "read_only": 1
+  },
+  {
+   "fieldname": "exception",
+   "fieldtype": "Text",
+   "label": "Exception",
+   "read_only": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2022-11-12 16:48:37.797232",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Submission Queue",
+ "naming_rule": "Random",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [
+  {
+   "color": "Blue",
+   "title": "Queued"
+  },
+  {
+   "color": "Red",
+   "title": "Failed"
+  },
+  {
+   "color": "Green",
+   "title": "Finished"
+  }
+ ]
+}

frappe/core/doctype/submission_queue/submission_queue.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2022, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from urllib.parse import quote
+
+from rq import get_current_job
+from rq.exceptions import NoSuchJobError
+from rq.job import Job
+
+import frappe
+from frappe import _
+from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
+from frappe.model.document import Document
+from frappe.monitor import add_data_to_monitor
+from frappe.utils import now, time_diff_in_seconds
+from frappe.utils.background_jobs import get_redis_conn
+from frappe.utils.data import cint
+
+
+class SubmissionQueue(Document):
+	@property
+	def created_at(self):
+		return self.creation
+
+	@property
+	def enqueued_by(self):
+		return self.owner
+
+	@property
+	def queued_doc(self):
+		return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname))
+
+	@staticmethod
+	def clear_old_logs(days=30):
+		from frappe.query_builder import Interval
+		from frappe.query_builder.functions import Now
+
+		table = frappe.qb.DocType("Submission Queue")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
+
+	def insert(self, to_be_queued_doc: Document, action: str):
+		self.to_be_queued_doc = to_be_queued_doc
+		self.action_for_queuing = action
+		super().insert(ignore_permissions=True)
+
+	def lock(self):
+		self.queued_doc.lock()
+
+	def unlock(self):
+		self.queued_doc.unlock()
+
+	def update_job_id(self, job_id):
+		frappe.db.set_value(
+			self.doctype,
+			self.name,
+			{"job_id": job_id},
+			update_modified=False,
+		)
+		frappe.db.commit()
+
+	def after_insert(self):
+		self.queue_action(
+			"background_submission",
+			to_be_queued_doc=self.queued_doc,
+			action_for_queuing=self.action_for_queuing,
+			timeout=600,
+			enqueue_after_commit=True,
+		)
+
+	def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str):
+		# Set the job id for that submission doctype
+		self.update_job_id(get_current_job().id)
+		_action = action_for_queuing.lower()
+		if _action == "update":
+			_action = "submit"
+
+		try:
+			getattr(to_be_queued_doc, _action)()
+			add_data_to_monitor(
+				doctype=to_be_queued_doc.doctype,
+				docname=to_be_queued_doc.name,
+				action=_action,
+				execution_time=time_diff_in_seconds(now(), self.created_at),
+				enqueued_by=self.enqueued_by,
+			)
+			values = {"status": "Finished"}
+		except Exception:
+			values = {"status": "Failed", "exception": frappe.get_traceback()}
+			frappe.db.rollback()
+
+		values["ended_at"] = now()
+		frappe.db.set_value(self.doctype, self.name, values, update_modified=False)
+		self.notify(values["status"], action_for_queuing)
+
+	def notify(self, submission_status: str, action: str):
+		if submission_status == "Failed":
+			doctype = self.doctype
+			docname = self.name
+			message = _("Submission of {0} {1} with action {2} failed")
+		else:
+			doctype = self.ref_doctype
+			docname = self.ref_docname
+			message = _("Submission of {0} {1} with action {2} completed successfully")
+
+		message = message.format(
+			frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action)
+		)
+		time_diff = time_diff_in_seconds(now(), self.created_at)
+		if cint(time_diff) <= 60:
+			frappe.publish_realtime(
+				"msgprint",
+				{
+					"message": message
+					+ f". View it <a href='/app/{quote(doctype.lower().replace(' ', '-'))}/{quote(docname)}'><b>here</b></a>",
+					"alert": True,
+					"indicator": "red" if submission_status == "Failed" else "green",
+				},
+				user=self.enqueued_by,
+			)
+		else:
+			notification_doc = {
+				"type": "Alert",
+				"document_type": doctype,
+				"document_name": docname,
+				"subject": message,
+			}
+
+			notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email")
+			enqueue_create_notification([notify_to], notification_doc)
+
+	def _unlock_reference_doc(self):
+		"""
+		Only execute if self.job_id is defined.
+		"""
+		try:
+			job = Job.fetch(self.job_id, connection=get_redis_conn())
+			status = job.get_status(refresh=True)
+			exc = job.exc_info
+		except NoSuchJobError:
+			exc = None
+			status = "failed"
+
+		if status in ("queued", "started"):
+			frappe.msgprint(_("Document in queue for execution!"))
+			return
+
+		self.queued_doc.unlock()
+		values = (
+			{"status": "Finished"} if status == "finished" else {"status": "Failed", "exception": exc}
+		)
+		frappe.db.set_value(self.doctype, self.name, values, update_modified=False)
+		frappe.msgprint(_("Document Unlocked"))
+
+	@frappe.whitelist()
+	def unlock_doc(self):
+		# NOTE: this can lead to some weird unlocking/locking behaviours.
+		# for example: hitting unlock on a submission could lead to unlocking of another submission
+		# of the same reference document.
+
+		if self.status != "Queued" and not self.job_id:
+			return
+
+		self._unlock_reference_doc()
+
+
+def queue_submission(doc: Document, action: str, alert: bool = True):
+	queue = frappe.new_doc("Submission Queue")
+	queue.state = "Queued"
+	queue.ref_doctype = doc.doctype
+	queue.ref_docname = doc.name
+	queue.insert(doc, action)
+
+	if alert:
+		frappe.msgprint(
+			_("Queued for Submission. You can track the progress over {0}.").format(
+				f"<a href='/app/submission-queue/{queue.name}'><b>here</b></a>"
+			),
+			indicator="green",
+			alert=True,
+		)
+
+
+@frappe.whitelist()
+def get_latest_submissions(doctype, docname):
+	# NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere
+	# hence assuming modified will be equal to creation for submission queue documents
+
+	dt = "Submission Queue"
+	filters = {"ref_doctype": doctype, "ref_docname": docname}
+	return {
+		"latest_submission": frappe.db.get_value(dt, filters),
+		"latest_failed_submission": frappe.db.get_value(dt, filters | {"status": "Failed"}),
+	}

frappe/core/doctype/submission_queue/test_submission_queue.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2022, Frappe Technologies and Contributors
+# See license.txt
+
+import time
+import typing
+
+import frappe
+from frappe.tests.utils import FrappeTestCase, timeout
+from frappe.utils.background_jobs import get_queue
+
+if typing.TYPE_CHECKING:
+	from rq.job import Job
+
+
+class TestSubmissionQueue(FrappeTestCase):
+	queue = get_queue(qtype="default")
+
+	@timeout(seconds=20)
+	def check_status(self, job: "Job", status, wait=True):
+		if wait:
+			while True:
+				if job.is_queued or job.is_started:
+					time.sleep(0.2)
+				else:
+					break
+		self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status)
+
+	def test_queue_operation(self):
+		from frappe.core.doctype.doctype.test_doctype import new_doctype
+		from frappe.core.doctype.submission_queue.submission_queue import queue_submission
+
+		if not frappe.db.table_exists("Test Submission Queue", cached=False):
+			doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True)
+			doc.insert()
+
+		d = frappe.new_doc("Test Submission Queue")
+		d.update({"some_fieldname": "Random"})
+		d.insert()
+
+		frappe.db.commit()
+		queue_submission(d, "submit")
+		frappe.db.commit()
+
+		# Waiting for execution
+		time.sleep(4)
+		submission_queue = frappe.get_last_doc("Submission Queue")
+
+		# Test queueing / starting
+		job = self.queue.fetch_job(submission_queue.job_id)
+		# Test completion
+		self.check_status(job, status="finished")

frappe/core/doctype/user/user.py

@@ -471,7 +471,7 @@ class User(Document):
 			frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False)
 
 		# set email
-		frappe.db.update("User", new_name, "email", new_name)
+		frappe.db.set_value("User", new_name, "email", new_name)
 
 	def append_roles(self, *roles):
 		"""Add roles to user"""

frappe/core/page/permission_manager/permission_manager.py

@@ -19,7 +19,6 @@ from frappe.permissions import (
 	setup_custom_perms,
 	update_permission_property,
 )
-from frappe.translate import send_translations
 from frappe.utils.user import get_users_with_role as _get_user_with_role
 
 not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"]
@@ -28,7 +27,6 @@ not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Tran
 @frappe.whitelist()
 def get_roles_and_doctypes():
 	frappe.only_for("System Manager")
-	send_translations(frappe.get_lang_dict("doctype", "DocPerm"))
 
 	active_domains = frappe.get_active_domains()
 

frappe/custom/doctype/customize_form/customize_form.js

@@ -149,6 +149,10 @@ frappe.ui.form.on("Customize Form", {
 			const is_autoname_autoincrement = frm.doc.autoname === "autoincrement";
 			frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement);
 			frm.set_df_property("autoname", "read_only", is_autoname_autoincrement);
+			frm.toggle_display(
+				["queue_in_background"],
+				frappe.get_meta(frm.doc.doc_type).is_submittable || 0
+			);
 		}
 
 		frm.events.setup_export(frm);

frappe/custom/doctype/customize_form/customize_form.json

@@ -20,6 +20,7 @@
   "track_views",
   "allow_auto_repeat",
   "allow_import",
+  "queue_in_background",
   "fields_section_break",
   "fields",
   "naming_section",
@@ -341,6 +342,12 @@
    "fieldtype": "Check",
    "label": "Make Attachments Public by Default"
   },
+  {
+   "default": "0",
+   "fieldname": "queue_in_background",
+   "fieldtype": "Check",
+   "label": "Queue in Background"
+  },
   {
    "fieldname": "default_view",
    "fieldtype": "Select",
@@ -367,7 +374,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2022-08-30 11:45:16.772277",
+ "modified": "2022-10-30 23:39:49.628093",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form",

frappe/custom/doctype/customize_form/customize_form.py

@@ -571,6 +571,7 @@ doctype_properties = {
 	"allow_copy": "Check",
 	"istable": "Check",
 	"quick_entry": "Check",
+	"queue_in_background": "Check",
 	"editable_grid": "Check",
 	"max_attachments": "Int",
 	"make_attachments_public": "Check",

frappe/database/database.py

@@ -7,7 +7,7 @@ import random
 import re
 import string
 import traceback
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from time import time
 
 from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
@@ -29,7 +29,8 @@ from frappe.exceptions import DoesNotExistError, ImplicitCommitError
 from frappe.model.utils.link_count import flush_local_link_count
 from frappe.query_builder.functions import Count
 from frappe.utils import cast as cast_fieldtype
-from frappe.utils import get_datetime, get_table_name, getdate, now, sbool
+from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool
+from frappe.utils.deprecations import deprecated
 
 IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE)
 INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*")
@@ -114,6 +115,17 @@ class Database:
 		self._cursor = self._conn.cursor()
 		frappe.local.rollback_observers = []
 
+		try:
+			if execution_timeout := get_query_execution_timeout():
+				self.set_execution_timeout(execution_timeout)
+		except Exception as e:
+			frappe.logger("database").warning(f"Couldn't set execution timeout {e}")
+
+	def set_execution_timeout(self, seconds: int):
+		"""Set session speicifc timeout on exeuction of statements.
+		If any statement takes more time it will be killed along with entire transaction."""
+		raise NotImplementedError
+
 	def use(self, db_name):
 		"""`USE` db_name."""
 		self._conn.select_db(db_name)
@@ -135,12 +147,11 @@ class Database:
 		self,
 		query: Query,
 		values: QueryValues = EmptyQueryValues,
+		*,
 		as_dict=0,
 		as_list=0,
-		formatted=0,
 		debug=0,
 		ignore_ddl=0,
-		as_utf8=0,
 		auto_commit=0,
 		update=None,
 		explain=False,
@@ -153,10 +164,8 @@ class Database:
 		:param values: Tuple / List / Dict of values to be escaped and substituted in the query.
 		:param as_dict: Return as a dictionary.
 		:param as_list: Always return as a list.
-		:param formatted: Format values like date etc.
 		:param debug: Print query and `EXPLAIN` in debug log.
 		:param ignore_ddl: Catch exception if table, column missing.
-		:param as_utf8: Encode values as UTF 8.
 		:param auto_commit: Commit after executing the query.
 		:param update: Update this dict to all rows (if returned `as_dict`).
 		:param run: Returns query without executing it if False.
@@ -264,13 +273,13 @@ class Database:
 
 		# scrub output if required
 		if as_dict:
-			ret = self.fetch_as_dict(formatted, as_utf8)
+			ret = self.fetch_as_dict()
 			if update:
 				for r in ret:
 					r.update(update)
 			return ret
-		elif as_list or as_utf8:
-			return self.convert_to_lists(self.last_result, formatted, as_utf8)
+		elif as_list:
+			return self.convert_to_lists(self.last_result)
 		return self.last_result
 
 	def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None:
@@ -377,56 +386,27 @@ class Database:
 		):
 			raise ImplicitCommitError("This statement can cause implicit commit")
 
-	def fetch_as_dict(self, formatted=0, as_utf8=0) -> list[frappe._dict]:
+	def fetch_as_dict(self) -> list[frappe._dict]:
 		"""Internal. Converts results to dict."""
 		result = self.last_result
-		ret = []
 		if result:
 			keys = [column[0] for column in self._cursor.description]
 
-		for r in result:
-			values = []
-			for value in r:
-				if as_utf8 and isinstance(value, str):
-					value = value.encode("utf-8")
-				values.append(value)
-
-			ret.append(frappe._dict(zip(keys, values)))
-		return ret
+		return [frappe._dict(zip(keys, row)) for row in result]
 
 	@staticmethod
 	def clear_db_table_cache(query):
 		if query and is_query_type(query, ("drop", "create")):
 			frappe.cache().delete_key("db_tables")
 
-	@staticmethod
-	def needs_formatting(result, formatted):
-		"""Returns true if the first row in the result has a Date, Datetime, Long Int."""
-		if result and result[0]:
-			for v in result[0]:
-				if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)):
-					return True
-				if formatted and isinstance(v, (int, float)):
-					return True
-
-		return False
-
 	def get_description(self):
 		"""Returns result metadata."""
 		return self._cursor.description
 
 	@staticmethod
-	def convert_to_lists(res, formatted=0, as_utf8=0):
+	def convert_to_lists(res):
 		"""Convert tuple output to lists (internal)."""
-		nres = []
-		for r in res:
-			nr = []
-			for val in r:
-				if as_utf8 and isinstance(val, str):
-					val = val.encode("utf-8")
-				nr.append(val)
-			nres.append(nr)
-		return nres
+		return [[value for value in row] for row in res]
 
 	def get(self, doctype, filters=None, as_dict=True, cache=False):
 		"""Returns `get_value` with fieldname='*'"""
@@ -786,13 +766,11 @@ class Database:
 		distinct=False,
 		limit=None,
 	):
-		field_objects = []
 		query = frappe.qb.engine.get_query(
 			table=doctype,
 			filters=filters,
 			orderby=order_by,
 			for_update=for_update,
-			field_objects=field_objects,
 			fields=fields,
 			distinct=distinct,
 			limit=limit,
@@ -828,10 +806,6 @@ class Database:
 			).run(debug=debug, run=run, as_dict=as_dict)
 		return {}
 
-	def update(self, *args, **kwargs):
-		"""Update multiple values. Alias for `set_value`."""
-		return self.set_value(*args, **kwargs)
-
 	def set_value(
 		self,
 		dt,
@@ -857,7 +831,6 @@ class Database:
 		:param modified_by: Set this user as `modified_by`.
 		:param update_modified: default True. Set as false, if you don't want to update the timestamp.
 		:param debug: Print the query in the developer / js console.
-		:param for_update: [DEPRECATED] This function now performs updates in single query, locking is not required.
 		"""
 		is_single_doctype = not (dn and dt != dn)
 		to_update = field if isinstance(field, dict) else {field: val}
@@ -897,30 +870,6 @@ class Database:
 		if dt in self.value_cache:
 			del self.value_cache[dt]
 
-	@staticmethod
-	def set(doc, field, val):
-		"""Set value in document. **Avoid**"""
-		doc.db_set(field, val)
-
-	def touch(self, doctype, docname):
-		"""Update the modified timestamp of this document."""
-		modified = now()
-		DocType = frappe.qb.DocType(doctype)
-		frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run()
-		return modified
-
-	@staticmethod
-	def set_temp(value):
-		"""Set a temperory value and return a key."""
-		key = frappe.generate_hash()
-		frappe.cache().hset("temp", key, value)
-		return key
-
-	@staticmethod
-	def get_temp(key):
-		"""Return the temperory value and delete it."""
-		return frappe.cache().hget("temp", key)
-
 	def set_global(self, key, val, user="__global"):
 		"""Save a global key value. Global values will be automatically set if they match fieldname."""
 		self.set_default(key, val, user)
@@ -1080,7 +1029,7 @@ class Database:
 		return getdate(date).strftime("%Y-%m-%d")
 
 	@staticmethod
-	def format_datetime(datetime):
+	def format_datetime(datetime):  # noqa: F811
 		if not datetime:
 			return FallBackDateTimeStr
 
@@ -1224,9 +1173,6 @@ class Database:
 		"""
 		return self.sql_ddl(f"truncate `{get_table_name(doctype)}`")
 
-	def clear_table(self, doctype):
-		return self.truncate(doctype)
-
 	def get_last_created(self, doctype):
 		last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc")
 		if last_record:
@@ -1342,3 +1288,28 @@ def savepoint(catch: type | tuple[type, ...] = Exception):
 		frappe.db.rollback(save_point=savepoint)
 	else:
 		frappe.db.release_savepoint(savepoint)
+
+
+def get_query_execution_timeout() -> int:
+	"""Get execution timeout based on current timeout in different contexts.
+
+	    HTTP requests: HTTP timeout or a default (300)
+	    Background jobs: Job timeout
+	Console/Commands: No timeout = 0.
+
+	    Note: Timeout adds 1.5x as "safety factor"
+	"""
+	from rq import get_current_job
+
+	if not frappe.conf.get("enable_db_statement_timeout"):
+		return 0
+
+	# Zero means no timeout, which is the default value in db.
+	timeout = 0
+	with suppress(Exception):
+		if getattr(frappe.local, "request", None):
+			timeout = frappe.conf.http_timeout or 300
+		elif job := get_current_job():
+			timeout = job.timeout
+
+	return int(cint(timeout) * 1.5)

frappe/database/mariadb/database.py

@@ -68,6 +68,10 @@ class MariaDBExceptionUtil:
 	def is_syntax_error(e: pymysql.Error) -> bool:
 		return e.args[0] == ER.PARSE_ERROR
 
+	@staticmethod
+	def is_statement_timeout(e: pymysql.Error) -> bool:
+		return e.args[0] == 1969
+
 	@staticmethod
 	def is_data_too_long(e: pymysql.Error) -> bool:
 		return e.args[0] == ER.DATA_TOO_LONG
@@ -102,6 +106,9 @@ class MariaDBConnectionUtil:
 	def create_connection(self):
 		return pymysql.connect(**self.get_connection_settings())
 
+	def set_execution_timeout(self, seconds: int):
+		self.sql("set session max_statement_time = %s", int(seconds))
+
 	def get_connection_settings(self) -> dict:
 		conn_settings = {
 			"host": self.host,

frappe/database/postgres/database.py

@@ -99,6 +99,10 @@ class PostgresExceptionUtil:
 	def is_duplicate_fieldname(e):
 		return getattr(e, "pgcode", None) == DUPLICATE_COLUMN
 
+	@staticmethod
+	def is_statement_timeout(e):
+		return PostgresDatabase.is_timedout(e) or isinstance(e, frappe.QueryTimeoutError)
+
 	@staticmethod
 	def is_data_too_long(e):
 		return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION
@@ -161,6 +165,10 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 
 		return conn
 
+	def set_execution_timeout(self, seconds: int):
+		# Postgres expects milliseconds as input
+		self.sql("set local statement_timeout = %s", int(seconds) * 1000)
+
 	def escape(self, s, percent=True):
 		"""Escape quotes and percent in given string."""
 		if isinstance(s, bytes):

frappe/database/query.py

@@ -1,3 +1,4 @@
+import itertools
 import operator
 import re
 from ast import literal_eval
@@ -10,11 +11,11 @@ from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
 
 import frappe
 from frappe import _
-from frappe.database.utils import is_pypika_function_object
+from frappe.database.utils import NestedSetHierarchy, is_pypika_function_object
 from frappe.model.db_query import get_timespan_date_range
 from frappe.query_builder import Criterion, Field, Order, Table, functions
 from frappe.query_builder.functions import Function, SqlFunctions
-from frappe.query_builder.utils import PseudoColumn
+from frappe.query_builder.utils import PseudoColumnMapper
 from frappe.utils.data import MARIADB_SPECIFIC_COMMENT
 
 if TYPE_CHECKING:
@@ -25,7 +26,6 @@ WORDS_PATTERN = re.compile(r"\w+")
 BRACKETS_PATTERN = re.compile(r"\(.*?\)|$")
 SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions]
 COMMA_PATTERN = re.compile(r",\s*(?![^()]*\))")
-TABLE_PATTERN = re.compile(r"`\btab\w+")
 
 
 def like(key: Field, value: str) -> frappe.qb:
@@ -164,8 +164,42 @@ def has_function(field):
 
 
 def table_from_string(table: str) -> "DocType":
-	table_name = table.split("`", maxsplit=1)[1].split(".")[0][3:]
-	return frappe.qb.DocType(table_name=table_name.replace("`", ""))
+	if frappe.db.db_type == "postgres":
+		table_name = table.split('"', maxsplit=1)[1].split(".")[0][3:].replace('"', "")
+	else:
+		table_name = table.split("`", maxsplit=1)[1].split(".")[0][3:].replace("`", "")
+	return frappe.qb.DocType(table_name=table_name)
+
+
+def get_nested_set_hierarchy_result(hierarchy: str, field: str, table: str):
+	ref_doctype = table
+	try:
+		lft, rgt = (
+			frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == field).run()[0]
+		)
+	except IndexError:
+		lft, rgt = None, None
+
+	if hierarchy in ("descendants of", "not descendants of"):
+		result = (
+			frappe.qb.from_(ref_doctype)
+			.select(Field("name"))
+			.where(Field("lft") > lft)
+			.where(Field("rgt") < rgt)
+			.orderby(Field("lft"), order=Order.asc)
+			.run()
+		)
+	else:
+		# Get ancestor elements of a DocType with a tree structure
+		result = (
+			frappe.qb.from_(ref_doctype)
+			.select(Field("name"))
+			.where(Field("lft") < lft)
+			.where(Field("rgt") > rgt)
+			.orderby(Field("lft"), order=Order.desc)
+			.run()
+		)
+	return result
 
 
 # default operators
@@ -190,7 +224,7 @@ OPERATOR_MAP: dict[str, Callable] = {
 	"between": func_between,
 	"is": func_is,
 	"timespan": func_timespan,
-	# TODO: Add support for nested set
+	"nested_set": NestedSetHierarchy,
 	# TODO: Add support for custom operators (WIP) - via filters_config hooks
 }
 
@@ -348,7 +382,24 @@ class Engine:
 			if not isinstance(key, str):
 				conditions = conditions.where(self.make_function_for_filters(key, value))
 				continue
+			# Nested set support
 			if isinstance(value, (list, tuple)):
+				if value[0] in self.OPERATOR_MAP["nested_set"]:
+					hierarchy, _field = value
+					result = get_nested_set_hierarchy_result(hierarchy, _field, table)
+					_operator = (
+						self.OPERATOR_MAP["not in"]
+						if hierarchy in ("not ancestors of", "not descendants of")
+						else self.OPERATOR_MAP["in"]
+					)
+					if result:
+						result = list(itertools.chain.from_iterable(result))
+						conditions = conditions.where(_operator(getattr(table, key), result))
+					else:
+						conditions = conditions.where(_operator(getattr(table, key), ("",)))
+					# Allow additional conditions
+					break
+
 				_operator = self.OPERATOR_MAP[value[0].casefold()]
 				_value = value[1] if value[1] else ("",)
 				conditions = conditions.where(_operator(getattr(table, key), _value))
@@ -416,11 +467,16 @@ class Engine:
 						if isinstance(operator_mapping, BuiltinFunctionType):
 							has_primitive_operator = True
 							field = operator_mapping(
-								*map(lambda field: Field(field.strip()), arg.split(_operator)),
+								*map(
+									lambda field: Field(field.strip())
+									if "`" not in field
+									else PseudoColumnMapper(field.strip()),
+									arg.split(_operator),
+								),
 							)
 
 				field = (
-					(Field(initial_fields) if "`" not in initial_fields else PseudoColumn(initial_fields))
+					(Field(initial_fields) if "`" not in initial_fields else PseudoColumnMapper(initial_fields))
 					if not has_primitive_operator
 					else field
 				)
@@ -500,18 +556,14 @@ class Engine:
 				alias = None
 				if " as " in field:
 					field, alias = field.split(" as ")
-				self.fieldname, linked_fieldname = field.split(".")
-				linked_field = frappe.get_meta(doctype, cached=True).get_field(self.fieldname)
-				try:
-					self.linked_doctype = linked_field.options
-				except AttributeError:
-					return fields
-				field = f"`tab{self.linked_doctype}`.`{linked_fieldname}`"
-				if alias:
-					field = f"{field} as {alias}"
-				_fields.append(field)
+				fieldname, linked_fieldname = field.split(".")
+				linked_doctype = frappe.get_meta(doctype).get_field(fieldname).options
 
-			return _fields
+				field = f"`tab{linked_doctype}`.`{linked_fieldname}`"
+				if alias:
+					field = f"{field} {alias}"
+			_fields.append(field)
+		return _fields
 
 	def sanitize_fields(self, fields: str | list | tuple):
 		is_mariadb = frappe.db.db_type == "mariadb"
@@ -531,21 +583,21 @@ class Engine:
 
 		return fields
 
-	def get_list_fields(self, fields: list) -> list:
+	def get_list_fields(self, table: str, fields: list) -> list:
 		updated_fields = []
 		if issubclass(type(fields), Criterion) or "*" in fields:
 			return fields
-		# fields = self.get_fieldnames_from_child_table(doctype=table, fields=fields)
+		fields = self.get_fieldnames_from_child_table(doctype=table, fields=fields)
 		for field in fields:
 			if not isinstance(field, Criterion) and field:
 				if " as " in field:
 					field, reference = field.split(" as ")
 					if "`" in field:
-						updated_fields.append(PseudoColumn(f"{field} as {reference}"))
+						updated_fields.append(PseudoColumnMapper(f"{field} {reference}"))
 					else:
 						updated_fields.append(Field(field.strip()).as_(reference))
 				elif "`" in str(field):
-					updated_fields.append(PseudoColumn(field.strip()))
+					updated_fields.append(PseudoColumnMapper(field.strip()))
 				else:
 					updated_fields.append(Field(field))
 		return updated_fields
@@ -554,16 +606,16 @@ class Engine:
 		if fields == "*":
 			return fields
 		if "`" in fields:
-			fields = PseudoColumn(fields)
+			fields = PseudoColumnMapper(fields)
 		if " as " in str(fields):
 			fields, reference = str(fields).split(" as ")
 			if "`" in str(fields):
-				fields = PseudoColumn(f"{fields} as {reference}")
+				fields = PseudoColumnMapper(f"{fields} {reference}")
 			else:
 				fields = Field(fields).as_(reference)
 		return fields
 
-	def set_fields(self, fields, **kwargs) -> list:
+	def set_fields(self, table: str, fields, **kwargs) -> list:
 		fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name"
 		fields = self.sanitize_fields(fields)
 		if isinstance(fields, list) and None in fields and Field not in fields:
@@ -591,7 +643,7 @@ class Engine:
 		if is_str:
 			fields = self.get_string_fields(fields)
 		if not is_str and fields:
-			fields = self.get_list_fields(fields)
+			fields = self.get_list_fields(table, fields)
 
 		# Need to check instance again since fields modified.
 		if not isinstance(fields, (list, tuple, set)):
@@ -600,51 +652,72 @@ class Engine:
 		fields.extend(function_objects)
 		return fields
 
-	def join_(self, criterion, fields, table, join):
+	def join_child_tables(
+		self,
+		criterion: Criterion,
+		join_type: str,
+		child_table: Table,
+		parent_table: Table,
+	) -> Criterion:
+		if self.joined_tables.get(join_type) != child_table:
+			criterion = getattr(criterion, join_type)(child_table).on(
+				(child_table.parent == parent_table.name)
+				& (child_table.parenttype == TAB_PATTERN.sub("", parent_table._table_name))
+			)
+			self.joined_tables[join_type] = child_table
+		return criterion
+
+	def join(self, criterion, fields, table, join_type):
 		"""Handles all join operations on criterion objects"""
 		has_join = False
-		joined_tables = {}
+		table_pattern = (
+			re.compile(r"`\btab\w+") if frappe.db.db_type == "mariadb" else re.compile(r'"\btab\w+')
+		)
+
+		def _update_pypika_fields(field):
+			if not is_pypika_function_object(field):
+				field = field if isinstance(field, (str, PseudoColumnMapper)) else field.get_sql()
+				if not table_pattern.search(str(field)):
+					if isinstance(field, PseudoColumnMapper):
+						field = field.get_sql()
+					return getattr(frappe.qb.DocType(table), field)
+				else:
+					return field
+			else:
+				field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args]
+				return field
 
 		if not isinstance(fields, Criterion):
 			for field in fields:
 				# Only perform this bit if foreign doctype in fields
 				if (
 					not is_pypika_function_object(field)
-					and str(field).startswith("`tab")
-					and (f"`tab{table}`" not in str(field))
+					and (str(field).startswith('"tab') or str(field).startswith("`tab"))
+					and (f"`tab{table}`" not in str(field) and f'tab{table}"' not in str(field))
 				):
 					has_join = True
-					table_to_join_on = table_from_string(str(field))
-					if joined_tables.get(join) != table_to_join_on:
-						criterion = getattr(criterion, join)(table_to_join_on).on(
-							getattr(table_to_join_on, "parent") == getattr(frappe.qb.DocType(table), "name")
-						)
-						joined_tables[join] = table_to_join_on
+					child_table = table_from_string(str(field))
+					parent_table = frappe.qb.DocType(table) if not isinstance(table, Table) else table
+					criterion = self.join_child_tables(
+						criterion=criterion,
+						join_type=join_type,
+						child_table=child_table,
+						parent_table=parent_table,
+					)
 
 			if has_join:
-
-				def _update_pypika_fields(field):
-					if not is_pypika_function_object(field):
-						field = field if isinstance(field, (str, PseudoColumn)) else field.get_sql()
-						if not TABLE_PATTERN.search(str(field)):
-							if isinstance(field, PseudoColumn):
-								field = field.get_sql()
-							return getattr(frappe.qb.DocType(table), field)
-						else:
-							return field
-					else:
-						field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args]
-						return field
-
 				fields = [_update_pypika_fields(field) for field in fields]
 
 		if len(self.tables) > 1:
-			primary_table = self.tables.pop(table)
-			for table_object in self.tables.values():
-				if joined_tables.get("left_join") != table_object:
-					criterion = getattr(criterion, join)(table_object).on(
-						table_object.parent == primary_table.name
-					)
+			parent_table = self.tables[table]
+			child_tables = list(self.tables.values())[1:]
+			for child_table in child_tables:
+				criterion = self.join_child_tables(
+					criterion,
+					join_type=join_type,
+					child_table=child_table,
+					parent_table=parent_table,
+				)
 
 		return criterion, fields
 
@@ -657,13 +730,16 @@ class Engine:
 	) -> MySQLQueryBuilder | PostgreSQLQueryBuilder:
 		# Clean up state before each query
 		self.tables = {}
+		self.joined_tables = {}
 		self.linked_doctype = None
 		self.fieldname = None
 
-		fields = self.set_fields(kwargs.get("field_objects") or fields, **kwargs)
 		criterion = self.build_conditions(table, filters, **kwargs)
-		join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join"
-		criterion, fields = self.join_(criterion=criterion, fields=fields, table=table, join=join)
+		fields = self.set_fields(table, fields, **kwargs)
+		join_type = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join"
+		criterion, fields = self.join(
+			criterion=criterion, fields=fields, table=table, join_type=join_type
+		)
 
 		if isinstance(fields, (list, tuple)):
 			query = criterion.select(*fields)

frappe/database/utils.py

@@ -18,6 +18,13 @@ QueryValues = tuple | list | dict | NoneType
 EmptyQueryValues = object()
 FallBackDateTimeStr = "0001-01-01 00:00:00.000000"
 
+NestedSetHierarchy = (
+	"ancestors of",
+	"descendants of",
+	"not ancestors of",
+	"not descendants of",
+)
+
 
 def is_query_type(query: str, query_type: str | tuple[str]) -> bool:
 	return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type)

frappe/desk/desk_page.py

@@ -2,7 +2,6 @@
 # License: MIT. See LICENSE
 
 import frappe
-from frappe.translate import send_translations
 
 
 @frappe.whitelist()
@@ -31,10 +30,6 @@ def getpage():
 	page = frappe.form_dict.get("name")
 	doc = get(page)
 
-	# load translations
-	if frappe.lang != "en":
-		send_translations(frappe.get_lang_dict("page", page))
-
 	frappe.response.docs.append(doc)
 
 

frappe/desk/doctype/bulk_update/bulk_update.py

@@ -3,8 +3,10 @@
 
 import frappe
 from frappe import _
+from frappe.core.doctype.submission_queue.submission_queue import queue_submission
 from frappe.model.document import Document
 from frappe.utils import cint
+from frappe.utils.scheduler import is_scheduler_inactive
 
 
 class BulkUpdate(Document):
@@ -44,8 +46,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):
 		try:
 			message = ""
 			if action == "submit" and doc.docstatus.is_draft():
-				doc.submit()
-				message = _("Submitting {0}").format(doctype)
+				if doc.meta.queue_in_background and not is_scheduler_inactive():
+					queue_submission(doc, action)
+					message = _("Queuing {0} for Submission").format(doctype)
+				else:
+					doc.submit()
+					message = _("Submitting {0}").format(doctype)
 			elif action == "cancel" and doc.docstatus.is_submitted():
 				doc.cancel()
 				message = _("Cancelling {0}").format(doctype)

frappe/desk/form/save.py

@@ -4,8 +4,10 @@
 import json
 
 import frappe
+from frappe.core.doctype.submission_queue.submission_queue import queue_submission
 from frappe.desk.form.load import run_onload
 from frappe.monitor import add_data_to_monitor
+from frappe.utils.scheduler import is_scheduler_inactive
 
 
 @frappe.whitelist()
@@ -16,8 +18,10 @@ def savedocs(doc, action):
 
 	# action
 	doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action]
-
 	if doc.docstatus == 1:
+		if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive():
+			queue_submission(doc, action)
+			return
 		doc.submit()
 	else:
 		doc.save()
@@ -27,7 +31,6 @@ def savedocs(doc, action):
 	send_updated_docs(doc)
 
 	add_data_to_monitor(doctype=doc.doctype, action=action)
-
 	frappe.msgprint(frappe._("Saved"), indicator="green", alert=True)
 
 

frappe/desk/query_report.py

@@ -16,7 +16,6 @@ from frappe.model.utils import render_include
 from frappe.modules import get_module_path, scrub
 from frappe.monitor import add_data_to_monitor
 from frappe.permissions import get_role_permissions
-from frappe.translate import send_translations
 from frappe.utils import (
 	cint,
 	cstr,
@@ -204,10 +203,6 @@ def get_script(report_name):
 	if not script:
 		script = "frappe.query_reports['%s']={}" % report_name
 
-	# load translations
-	if frappe.lang != "en":
-		send_translations(frappe.get_lang_dict("report", report_name))
-
 	return {
 		"script": render_include(script),
 		"html_format": html_format,

frappe/desk/reportview.py

@@ -91,7 +91,7 @@ def validate_args(data):
 def validate_fields(data):
 	wildcard = update_wildcard_field_param(data)
 
-	for field in data.fields or []:
+	for field in list(data.fields or []):
 		fieldname = extract_fieldname(field)
 		if is_standard(fieldname):
 			continue

frappe/email/doctype/newsletter/test_newsletter.py

@@ -74,7 +74,7 @@ class TestNewsletterMixin:
 				).insert(ignore_if_duplicate=True)
 			except Exception:
 				frappe.db.rollback(save_point=savepoint)
-				frappe.db.update(doctype, email_filters, "unsubscribed", 0)
+				frappe.db.set_value(doctype, email_filters, "unsubscribed", 0)
 
 			frappe.db.release_savepoint(savepoint)
 

frappe/email/oauth.py

@@ -132,19 +132,18 @@ def oauth_access(email_account: str, service: str):
 	if not service:
 		frappe.throw(frappe._("No Service is selected. Please select one and try again!"))
 
-	doctype = "Email Account"
-
 	if service == "GMail":
-		return authorize_google_access(email_account, doctype)
+		return authorize_google_access(email_account)
 
 	raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.")
 
 
-def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None):
+def authorize_google_access(email_account: str, code: str = None):
 	"""Facilitates google oauth for email.
-	This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url
+	This is invoked 2 times - first time when user clicks `Authorize API Access` for getting the authorization url
 	and second time for setting the refresh and access token in db when google redirects back with oauth code."""
 
+	doctype = "Email Account"
 	oauth_obj = GoogleOAuth("mail")
 
 	if not code:

frappe/installer.py

@@ -402,7 +402,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
 
 			if not dry_run:
 				if doctype.issingle:
-					frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True)
+					frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True, force=True)
 				else:
 					drop_doctypes.append(doctype.name)
 
@@ -460,7 +460,7 @@ def _delete_doctypes(doctypes: list[str], dry_run: bool) -> None:
 	for doctype in set(doctypes):
 		print(f"* dropping Table for '{doctype}'...")
 		if not dry_run:
-			frappe.delete_doc("DocType", doctype, ignore_on_trash=True)
+			frappe.delete_doc("DocType", doctype, ignore_on_trash=True, force=True)
 			frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{doctype}`")
 
 

frappe/model/base_document.py

@@ -100,12 +100,7 @@ class BaseDocument:
 		if d.get("doctype"):
 			self.doctype = d["doctype"]
 
-		self._table_fieldnames = (
-			d["_table_fieldnames"]  # from cache
-			if "_table_fieldnames" in d
-			else {df.fieldname for df in self._get_table_fields()}
-		)
-
+		self._table_fieldnames = {df.fieldname for df in self._get_table_fields()}
 		self.update(d)
 		self.dont_update_if_missing = []
 

frappe/model/db_query.py

@@ -13,7 +13,7 @@ import frappe.permissions
 import frappe.share
 from frappe import _
 from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
-from frappe.database.utils import FallBackDateTimeStr
+from frappe.database.utils import FallBackDateTimeStr, NestedSetHierarchy
 from frappe.model import optional_fields
 from frappe.model.meta import get_table_columns
 from frappe.model.utils.user_settings import get_user_settings, update_user_settings
@@ -568,21 +568,14 @@ class DatabaseQuery:
 		can_be_null = True
 
 		# prepare in condition
-		if f.operator.lower() in (
-			"ancestors of",
-			"descendants of",
-			"not ancestors of",
-			"not descendants of",
-		):
+		if f.operator.lower() in NestedSetHierarchy:
 			values = f.value or ""
 
 			# TODO: handle list and tuple
 			# if not isinstance(values, (list, tuple)):
 			# 	values = values.split(",")
-
 			field = meta.get_field(f.fieldname)
 			ref_doctype = field.options if field else f.doctype
-
 			lft, rgt = "", ""
 			if f.value:
 				lft, rgt = frappe.db.get_value(ref_doctype, f.value, ["lft", "rgt"])

frappe/model/delete_doc.py

@@ -92,6 +92,8 @@ def delete_doc(
 
 			else:
 				doc = frappe.get_doc(doctype, name)
+				if not (doc.custom or frappe.conf.developer_mode or frappe.flags.in_patch or force):
+					frappe.throw(_("Standard DocType can not be deleted."))
 
 				update_flags(doc, flags, ignore_permissions)
 				check_permission_and_not_submitted(doc)

frappe/model/document.py

@@ -120,6 +120,10 @@ class Document(BaseDocument):
 			# incorrect arguments. let's not proceed.
 			raise ValueError("Illegal arguments")
 
+	@property
+	def is_locked(self):
+		return file_lock.lock_exists(self.get_signature())
+
 	@staticmethod
 	def whitelist(fn):
 		"""Decorator: Whitelist method to be called remotely via REST API."""
@@ -142,9 +146,14 @@ class Document(BaseDocument):
 			self._fix_numeric_types()
 
 		else:
+			get_value_kwargs = {"for_update": self.flags.for_update, "as_dict": True}
+			if not isinstance(self.name, (dict, list)):
+				get_value_kwargs["order_by"] = None
+
 			d = frappe.db.get_value(
-				self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update
+				doctype=self.doctype, filters=self.name, fieldname="*", **get_value_kwargs
 			)
+
 			if not d:
 				frappe.throw(
 					_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError
@@ -245,7 +254,6 @@ class Document(BaseDocument):
 		self._set_defaults()
 		self.set_user_and_timestamp()
 		self.set_docstatus()
-		self.load_doc_before_save()
 		self.check_if_latest()
 		self._validate_links()
 		self.check_permission("create")
@@ -296,6 +304,10 @@ class Document(BaseDocument):
 				follow_document(self.doctype, self.name, frappe.session.user)
 		return self
 
+	def check_if_locked(self):
+		if self.creation and self.is_locked:
+			raise frappe.DocumentLockedError
+
 	def save(self, *args, **kwargs):
 		"""Wrapper for _save"""
 		return self._save(*args, **kwargs)
@@ -322,11 +334,11 @@ class Document(BaseDocument):
 		if self.get("__islocal") or not self.get("name"):
 			return self.insert()
 
+		self.check_if_locked()
 		self.check_permission("write", "save")
 
 		self.set_user_and_timestamp()
 		self.set_docstatus()
-		self.load_doc_before_save()
 		self.check_if_latest()
 		self.set_parent_in_children()
 		self.set_name_in_children()
@@ -746,10 +758,13 @@ class Document(BaseDocument):
 		Will also validate document transitions (Save > Submit > Cancel) calling
 		`self.check_docstatus_transition`."""
 
-		self._action = "save"
-		previous = self.get_doc_before_save()
+		self.load_doc_before_save(raise_exception=True)
 
-		if not previous or self.meta.get("is_virtual"):
+		self._action = "save"
+		previous = self._doc_before_save
+
+		# previous is None for new document insert
+		if not previous:
 			self.check_docstatus_transition(0)
 			return
 
@@ -1048,7 +1063,7 @@ class Document(BaseDocument):
 
 		self.set_title_field()
 
-	def load_doc_before_save(self):
+	def load_doc_before_save(self, *, raise_exception: bool = False):
 		"""load existing document from db before saving"""
 
 		self._doc_before_save = None
@@ -1059,6 +1074,9 @@ class Document(BaseDocument):
 		try:
 			self._doc_before_save = frappe.get_doc(self.doctype, self.name, for_update=True)
 		except frappe.DoesNotExistError:
+			if raise_exception:
+				raise
+
 			frappe.clear_last_message()
 
 	def run_post_save_methods(self):

frappe/model/naming.py

@@ -235,13 +235,14 @@ def set_naming_from_document_naming_rule(doc):
 	if doc.doctype in log_types:
 		return
 
-	# ignore_ddl if naming is not yet bootstrapped
-	for d in frappe.get_all(
+	document_naming_rules = frappe.cache_manager.get_doctype_map(
 		"Document Naming Rule",
-		dict(document_type=doc.doctype, disabled=0),
+		doc.doctype,
+		filters={"document_type": doc.doctype, "disabled": 0},
 		order_by="priority desc",
-		ignore_ddl=True,
-	):
+	)
+
+	for d in document_naming_rules:
 		frappe.get_cached_doc("Document Naming Rule", d.name).apply(doc)
 		if doc.name:
 			break
@@ -277,7 +278,7 @@ def make_autoname(key="", doctype="", doc=""):
 	                DE/09/01/00001 where 09 is the year, 01 is the month and 00001 is the series
 	"""
 	if key == "hash":
-		return frappe.generate_hash(doctype, 10)
+		return frappe.generate_hash(length=10)
 
 	series = NamingSeries(key)
 	return series.generate_next_name(doc)

frappe/model/workflow.py

@@ -1,12 +1,17 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import json
+from typing import TYPE_CHECKING, Union
 
 import frappe
 from frappe import _
 from frappe.model.docstatus import DocStatus
 from frappe.utils import cint
 
+if TYPE_CHECKING:
+	from frappe.model.document import Document
+	from frappe.workflow.doctype.workflow.workflow import Workflow
+
 
 class WorkflowStateError(frappe.ValidationError):
 	pass
@@ -32,20 +37,22 @@ def get_workflow_name(doctype):
 
 
 @frappe.whitelist()
-def get_transitions(doc, workflow=None, raise_exception=False):
+def get_transitions(
+	doc: Union["Document", str, dict], workflow: "Workflow" = None, raise_exception: bool = False
+) -> list[dict]:
 	"""Return list of possible transitions for the given doc"""
-	doc = frappe.get_doc(frappe.parse_json(doc))
+	from frappe.model.document import Document
+
+	if not isinstance(doc, Document):
+		doc = frappe.get_doc(frappe.parse_json(doc))
+		doc.load_from_db()
 
 	if doc.is_new():
 		return []
 
-	doc.load_from_db()
+	doc.check_permission("read")
 
-	frappe.has_permission(doc, "read", throw=True)
-	roles = frappe.get_roles()
-
-	if not workflow:
-		workflow = get_workflow(doc.doctype)
+	workflow = workflow or get_workflow(doc.doctype)
 	current_state = doc.get(workflow.workflow_state_field)
 
 	if not current_state:
@@ -55,11 +62,14 @@ def get_transitions(doc, workflow=None, raise_exception=False):
 			frappe.throw(_("Workflow State not set"), WorkflowStateError)
 
 	transitions = []
+	roles = frappe.get_roles()
+
 	for transition in workflow.transitions:
 		if transition.state == current_state and transition.allowed in roles:
 			if not is_transition_condition_satisfied(transition, doc):
 				continue
 			transitions.append(transition.as_dict())
+
 	return transitions
 
 
@@ -79,7 +89,7 @@ def get_workflow_safe_globals():
 	)
 
 
-def is_transition_condition_satisfied(transition, doc):
+def is_transition_condition_satisfied(transition, doc) -> bool:
 	if not transition.condition:
 		return True
 	else:
@@ -198,7 +208,7 @@ def validate_workflow(doc):
 			)
 
 
-def get_workflow(doctype):
+def get_workflow(doctype) -> "Workflow":
 	return frappe.get_doc("Workflow", get_workflow_name(doctype))
 
 

frappe/patches/v11_0/remove_skip_for_doctype.py

@@ -60,7 +60,7 @@ def execute():
 					# Maintain sequence (name, user, allow, for_value, applicable_for, apply_to_all_doctypes, creation, modified)
 					new_user_permissions_list.append(
 						(
-							frappe.generate_hash("", 10),
+							frappe.generate_hash(length=10),
 							user_permission.user,
 							user_permission.allow,
 							user_permission.for_value,

frappe/patches/v12_0/move_email_and_phone_to_child_table.py

@@ -27,7 +27,7 @@ def execute():
 			email_values.append(
 				(
 					1,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.email_id,
 					"email_ids",
 					"Contact",
@@ -44,7 +44,7 @@ def execute():
 			phone_values.append(
 				(
 					phone_counter,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.phone,
 					"phone_nos",
 					"Contact",
@@ -63,7 +63,7 @@ def execute():
 			phone_values.append(
 				(
 					phone_counter,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.mobile_no,
 					"phone_nos",
 					"Contact",

frappe/patches/v12_0/setup_tags.py

@@ -28,7 +28,7 @@ def execute():
 
 				tag_list.append((tag.strip(), time, time, "Administrator"))
 
-				tag_link_name = frappe.generate_hash(_user_tags.name + tag.strip() + doctype.name, 10)
+				tag_link_name = frappe.generate_hash(length=10)
 				tag_links.append(
 					(tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, "Administrator")
 				)

frappe/patches/v13_0/queryreport_columns.py

@@ -15,4 +15,4 @@ def execute():
 		if isinstance(data, list):
 			# double escape braces
 			jstr = f'{{"columns":{jstr}}}'
-			frappe.db.update("Report", record["name"], "json", jstr)
+			frappe.db.set_value("Report", record["name"], "json", jstr)

frappe/patches/v14_0/remove_db_aggregation.py

@@ -32,4 +32,4 @@ def execute():
 		for agg in ["avg", "max", "min", "sum"]:
 			script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script)
 
-		frappe.db.update("Server Script", name, "script", script)
+		frappe.db.set_value("Server Script", name, "script", script)

frappe/public/js/frappe/form/form.js

@@ -450,6 +450,7 @@ frappe.ui.form.Form = class FrappeForm {
 				.toggleClass("cancelled-form", this.doc.docstatus === 2);
 
 			this.show_conflict_message();
+			this.show_submission_queue_banner();
 
 			if (frappe.boot.read_only) {
 				this.disable_form();
@@ -2036,6 +2037,83 @@ frappe.ui.form.Form = class FrappeForm {
 			.filter((user) => !["Administrator", frappe.session.user].includes(user))
 			.filter(Boolean);
 	}
+
+	show_submission_queue_banner() {
+		let wrapper = this.layout.wrapper.find(".submission-queue-banner");
+
+		if (
+			!(
+				this.meta.is_submittable &&
+				this.meta.queue_in_background &&
+				!this.doc.__islocal &&
+				this.doc.docstatus === 0
+			)
+		) {
+			if (wrapper.length) {
+				wrapper.hide();
+				wrapper.html("");
+			}
+
+			return;
+		}
+
+		if (!wrapper.length) {
+			wrapper = $('<div class="submission-queue-banner form-message yellow">');
+			this.layout.wrapper.prepend(wrapper);
+		}
+
+		frappe
+			.call({
+				method: "frappe.core.doctype.submission_queue.submission_queue.get_latest_submissions",
+				args: { doctype: this.doctype, docname: this.docname },
+			})
+			.then((r) => {
+				if (r.message.latest_submission) {
+					// if we are here that means some submission(s) were queued and are in queued/failed state
+					let col_width = 4;
+					let failed_link = "";
+					let submission_label = __("Previous Submission");
+
+					if (r.message.latest_failed_submission) {
+						if (r.message.latest_failed_submission !== r.message.latest_submission) {
+							col_width = 3;
+							failed_link = `<div class="col-md-3">
+								<a href='/app/submission-queue/${r.message.latest_failed_submission}'>${__(
+								"Previous Falied Submission"
+							)}</a>
+							</div>`;
+						} else {
+							submission_label = __("Previous Falied Submission");
+						}
+					}
+
+					let html = `
+				<div class="row">
+					<div class="col-md-${col_width}">
+						<strong>${__("Submission Status:")}</strong>
+					</div>
+					<div class="col-md-${col_width}">
+						<a href='/app/submission-queue/${r.message.latest_submission}'>${submission_label}</a>
+					</div>
+					${failed_link}
+					<div class="col-md-${col_width}">
+						<a href='/app/submission-queue?ref_doctype=${encodeURIComponent(
+							this.doctype
+						)}&ref_docname=${encodeURIComponent(this.docname)}'>${__(
+						"All Submissions"
+					)}</a>
+					</div>
+				</div>
+				`;
+
+					wrapper.show();
+					wrapper.html(html);
+				} else {
+					wrapper.hide();
+					wrapper.html("");
+				}
+			});
+	}
 };
 
 frappe.validated = 0;

frappe/public/js/frappe/form/grid_row.js

@@ -944,19 +944,21 @@ export default class GridRow {
 				vertical = false;
 				horizontal = false;
 			})
-			.on("click", function () {
+			.on("click", function (event) {
 				if (frappe.ui.form.editable_row !== me) {
 					var out = me.toggle_editable_row();
 				}
 				var col = this;
 				let first_input_field = $(col).find('input[type="Text"]:first');
-
-				first_input_field.length && on_input_focus(first_input_field);
-
 				first_input_field.trigger("focus");
-				first_input_field.one("blur", () => (input_in_focus = false));
 
-				first_input_field.data("fieldtype") == "Date" && handle_date_picker();
+				if (event.pointerType == "touch") {
+					first_input_field.length && on_input_focus(first_input_field);
+
+					first_input_field.one("blur", () => (input_in_focus = false));
+
+					first_input_field.data("fieldtype") == "Date" && handle_date_picker();
+				}
 
 				return out;
 			});

frappe/public/js/frappe/list/list_view.js

@@ -1179,7 +1179,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 		this.$result.on("click", ".list-row, .image-view-header, .file-header", (e) => {
 			const $target = $(e.target);
 			// tick checkbox if Ctrl/Meta key is pressed
-			if (e.ctrlKey || (e.metaKey && !$target.is("a"))) {
+			if ((e.ctrlKey || e.metaKey) && !$target.is("a")) {
 				const $list_row = $(e.currentTarget);
 				const $check = $list_row.find(".list-row-checkbox");
 				$check.prop("checked", !$check.prop("checked"));

frappe/public/js/frappe/list/list_view_select.js

@@ -315,7 +315,7 @@ frappe.views.ListViewSelect = class ListViewSelect {
 		accounts.forEach((account) => {
 			let email_account =
 				account.email_id == "All Accounts" ? "All Accounts" : account.email_account;
-			let route = `/app/communication/inbox/${email_account}`;
+			let route = `/app/communication/view/inbox/${email_account}`;
 			let display_name = ["All Accounts", "Sent Mail", "Spam", "Trash"].includes(
 				account.email_id
 			)

frappe/public/js/frappe/ui/keyboard.js

@@ -331,7 +331,7 @@ function close_grid_and_dialog() {
 	}
 
 	// close open dialog
-	if (cur_dialog && !cur_dialog.no_cancel_flag) {
+	if (cur_dialog && !cur_dialog.no_cancel_flag && !cur_dialog.static) {
 		cur_dialog.cancel();
 		return false;
 	}

frappe/public/js/frappe/ui/page.js

@@ -327,13 +327,14 @@ frappe.ui.Page = class Page {
 
 	//--- Menu --//
 
-	add_menu_item(label, click, standard, shortcut) {
+	add_menu_item(label, click, standard, shortcut, show_parent) {
 		return this.add_dropdown_item({
 			label,
 			click,
 			standard,
 			parent: this.menu,
 			shortcut,
+			show_parent,
 		});
 	}
 
@@ -424,7 +425,7 @@ frappe.ui.Page = class Page {
 		icon = null,
 	}) {
 		if (show_parent) {
-			parent.parent().removeClass("hide");
+			parent.parent().removeClass("hide hidden-xl");
 		}
 
 		let $link = this.is_in_group_button_dropdown(parent, "li > a.grey-link > span", label);
@@ -600,6 +601,14 @@ frappe.ui.Page = class Page {
 			let response = action();
 			me.btn_disable_enable(btn, response);
 		};
+		// Add actions as menu item in Mobile View
+		let menu_item_label = group ? `${group} > ${label}` : label;
+		let menu_item = this.add_menu_item(menu_item_label, _action, false, false, false);
+		menu_item.parent().addClass("hidden-xl");
+		if (this.menu_btn_group.hasClass("hide")) {
+			this.menu_btn_group.removeClass("hide").addClass("hidden-xl");
+		}
+
 		if (group) {
 			var $group = this.get_or_add_inner_group_button(group);
 			$(this.inner_toolbar).removeClass("hide");

frappe/public/js/frappe/ui/toolbar/about.js

@@ -19,6 +19,8 @@ frappe.ui.misc.about = function () {
 						Facebook: <a href='https://facebook.com/erpnext' target='_blank'>https://facebook.com/erpnext</a></p>
 					<p><i class='fa fa-twitter fa-fw'></i>
 						Twitter: <a href='https://twitter.com/erpnext' target='_blank'>https://twitter.com/erpnext</a></p>
+					<p><i class='fa fa-youtube fa-fw'></i>
+						YouTube: <a href='https://www.youtube.com/@erpnextofficial' target='_blank'>https://www.youtube.com/@erpnextofficial</a></p>
 					<hr>
 					<h4>${__("Installed Apps")}</h4>
 					<div id='about-app-versions'>${__("Loading versions...")}</div>

frappe/public/js/frappe/utils/utils.js

@@ -1383,7 +1383,7 @@ Object.assign(frappe.utils, {
 			: "";
 
 		return $(`<div class="summary-item">
-			<span class="summary-label">${summary.label}</span>
+			<span class="summary-label">${__(summary.label)}</span>
 			<div class="summary-value ${color}">${value}</div>
 		</div>`);
 	},

frappe/public/js/frappe/views/breadcrumbs.js

@@ -40,7 +40,6 @@ frappe.breadcrumbs = {
 				type: type,
 			};
 		}
-
 		this.all[frappe.breadcrumbs.current_page()] = obj;
 		this.update();
 	},
@@ -137,13 +136,13 @@ frappe.breadcrumbs = {
 		const doctype_meta = frappe.get_doc("DocType", doctype);
 		if (
 			(doctype === "User" && !frappe.user.has_role("System Manager")) ||
-			(doctype_meta && doctype_meta.issingle)
+			doctype_meta?.issingle
 		) {
 			// no user listview for non-system managers and single doctypes
 		} else {
 			let route;
 			const doctype_route = frappe.router.slug(frappe.router.doctype_layout || doctype);
-			if (doctype_meta.is_tree) {
+			if (doctype_meta?.is_tree) {
 				let view = frappe.model.user_settings[doctype].last_view || "Tree";
 				route = `${doctype_route}/view/${view}`;
 			} else {

frappe/public/js/frappe/views/reports/query_report.js

@@ -1067,7 +1067,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				{
 					fieldname: "sb_1",
 					fieldtype: "Section Break",
-					label: "Y axis",
+					label: "Y Axis",
 				},
 				{
 					fieldname: "y_axis_fields",

frappe/query_builder/builder.py

@@ -47,6 +47,8 @@ class Base:
 class MariaDB(Base, MySQLQuery):
 	Field = terms.Field
 
+	_BuilderClasss = MySQLQueryBuilder
+
 	@classmethod
 	def _builder(cls, *args, **kwargs) -> "MySQLQueryBuilder":
 		return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs)
@@ -70,6 +72,8 @@ class Postgres(Base, PostgreSQLQuery):
 	# they are two different objects. The quick fix used here is to replace the
 	# Field names in the "Field" function.
 
+	_BuilderClasss = PostgreSQLQueryBuilder
+
 	@classmethod
 	def _builder(cls, *args, **kwargs) -> "PostgreSQLQueryBuilder":
 		return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs)

frappe/query_builder/utils.py

@@ -12,6 +12,16 @@ from frappe.query_builder.terms import NamedParameterWrapper
 from .builder import MariaDB, Postgres
 
 
+class PseudoColumnMapper(PseudoColumn):
+	def __init__(self, name: str) -> None:
+		super().__init__(name)
+
+	def get_sql(self, **kwargs):
+		if frappe.db.db_type == "postgres":
+			self.name = self.name.replace("`", '"')
+		return self.name
+
+
 class db_type_is(Enum):
 	MARIADB = "mariadb"
 	POSTGRES = "postgres"
@@ -102,8 +112,7 @@ def patch_query_execute():
 				raise frappe.PermissionError("Only SELECT SQL allowed in scripting")
 		return query, param_collector.get_parameters()
 
-	query_class = get_attr(str(frappe.qb).split("'")[1])
-	builder_class = get_type_hints(query_class._builder).get("return")
+	builder_class = frappe.qb._BuilderClasss
 
 	if not builder_class:
 		raise BuilderIdentificationFailed

frappe/templates/includes/navbar/navbar_items.html

@@ -3,7 +3,7 @@
 
 {% if parent %}
 
-{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%}
+{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
 <li class="nav-item dropdown {% if submenu %} dropdown-submenu {% endif %}">
 	<a class="nav-link dropdown-toggle" href="#" id="{{ dropdown_id }}" role="button"
 		data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
@@ -16,7 +16,7 @@
 	</ul>
 </li>
 {% else %}
-{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%}
+{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
 <li class="dropdown {% if submenu %} dropdown-submenu {% endif %}">
 	<a class="dropdown-item dropdown-toggle" href="#" id="{{ dropdown_id }}" role="button"
 		data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">

frappe/templates/includes/navbar/navbar_search.html

@@ -2,8 +2,8 @@
 	<li>
 		<form action='/search'>
 		<input name='q' class='form-control navbar-search' type='text'
-			value='{{ frappe.form_dict.q or ''}}'
+			value='{{ frappe.form_dict.q|e or ''}}'
 			{% if not frappe.form_dict.q%}placeholder="{{ _("Search...") }}"{% endif %}>
 		</form>
 	</li>
-{% endif %}
+{% endif %}

frappe/test_runner.py

@@ -86,7 +86,6 @@ def main(
 			frappe.utils.scheduler.disable_scheduler()
 
 		set_test_email_config()
-		frappe.conf.update({"bench_id": "test_bench", "use_rq_auth": False})
 
 		if not frappe.flags.skip_before_tests:
 			if verbose:

frappe/tests/test_db.py

@@ -11,13 +11,13 @@ import frappe
 from frappe.core.utils import find
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.database import savepoint
-from frappe.database.database import Database
+from frappe.database.database import Database, get_query_execution_timeout
 from frappe.database.utils import FallBackDateTimeStr
 from frappe.query_builder import Field
 from frappe.query_builder.functions import Concat_ws
 from frappe.tests.test_query_builder import db_type_is, run_only_if
 from frappe.tests.utils import FrappeTestCase
-from frappe.utils import add_days, cint, now, random_string
+from frappe.utils import add_days, cint, now, random_string, set_request
 from frappe.utils.testutils import clear_custom_fields
 
 
@@ -36,6 +36,34 @@ class TestDB(FrappeTestCase):
 	def test_get_database_size(self):
 		self.assertIsInstance(frappe.db.get_database_size(), (float, int))
 
+	def test_db_statement_execution_timeout(self):
+		frappe.db.set_execution_timeout(2)
+		# Setting 0 means no timeout.
+		self.addCleanup(frappe.db.set_execution_timeout, 0)
+
+		try:
+			savepoint = "statement_timeout"
+			frappe.db.savepoint(savepoint)
+			frappe.db.multisql(
+				{
+					"mariadb": "select sleep(10)",
+					"postgres": "select pg_sleep(10)",
+				}
+			)
+		except Exception as e:
+			self.assertTrue(frappe.db.is_statement_timeout(e), f"exepcted {e} to be timeout error")
+			frappe.db.rollback(save_point=savepoint)
+		else:
+			frappe.db.rollback(save_point=savepoint)
+			self.fail("Long running queries not timing out")
+
+	@patch.dict(frappe.conf, {"http_timeout": 20, "enable_db_statement_timeout": 1})
+	def test_db_timeout_computation(self):
+		set_request(method="GET", path="/")
+		self.assertEqual(get_query_execution_timeout(), 30)
+		frappe.local.request = None
+		self.assertEqual(get_query_execution_timeout(), 0)
+
 	def test_get_value(self):
 		self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator")
@@ -763,23 +791,6 @@ class TestDBSetValue(FrappeTestCase):
 		cached_doc = frappe.get_cached_doc(self.todo2.doctype, self.todo2.name)
 		self.assertEqual(cached_doc.description, description)
 
-	def test_update_alias(self):
-		args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`")
-		kwargs = {
-			"for_update": False,
-			"modified": None,
-			"modified_by": None,
-			"update_modified": True,
-			"debug": False,
-		}
-
-		self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update))
-
-		with patch.object(Database, "set_value") as set_value:
-			frappe.db.update(*args, **kwargs)
-			set_value.assert_called_once()
-			set_value.assert_called_with(*args, **kwargs)
-
 	@classmethod
 	def tearDownClass(cls):
 		frappe.db.rollback()

frappe/tests/test_document.py

@@ -411,6 +411,19 @@ class TestDocument(FrappeTestCase):
 		todo.save()
 		self.assertEqual(todo.notify_update.call_count, 1)
 
+	def test_error_on_saving_new_doc_with_name(self):
+		"""Trying to save a new doc with name should raise DoesNotExistError"""
+
+		doc = frappe.get_doc(
+			{
+				"doctype": "ToDo",
+				"description": "this should raise frappe.DoesNotExistError",
+				"name": "lets-trick-doc-save",
+			}
+		)
+
+		self.assertRaises(frappe.DoesNotExistError, doc.save)
+
 
 class TestDocumentWebView(FrappeTestCase):
 	def get(self, path, user="Guest"):

frappe/tests/test_document_locks.py

@@ -16,3 +16,23 @@ class TestDocumentLocks(FrappeTestCase):
 		todo_1.lock()
 		self.assertRaises(frappe.DocumentLockedError, todo.lock)
 		todo_1.unlock()
+
+	def test_operations_on_locked_documents(self):
+		todo = frappe.get_doc(dict(doctype="ToDo", description="testing operations")).insert()
+		todo.lock()
+
+		with self.assertRaises(frappe.DocumentLockedError):
+			todo.description = "Random"
+			todo.save()
+
+		# Checking for persistant locks across all instances.
+		doc = frappe.get_doc("ToDo", todo.name)
+		self.assertEqual(doc.is_locked, True)
+
+		with self.assertRaises(frappe.DocumentLockedError):
+			doc.description = "Random"
+			doc.save()
+
+		doc.unlock()
+		self.assertEqual(doc.is_locked, False)
+		self.assertEqual(todo.is_locked, False)

frappe/tests/test_email.py

@@ -310,6 +310,20 @@ class TestEmail(FrappeTestCase):
 			email_account.enable_incoming = False
 
 
+class TestVerifiedRequests(FrappeTestCase):
+	def test_round_trip(self):
+		from frappe.utils import set_request
+		from frappe.utils.verified_command import get_signed_params, verify_request
+
+		test_cases = [{"xyz": "abc"}, {"email": "a@b.com", "user": "xyz"}]
+
+		for params in test_cases:
+			signed_url = get_signed_params(params)
+			set_request(method="GET", path="?" + signed_url)
+			self.assertTrue(verify_request())
+		frappe.local.request = None
+
+
 if __name__ == "__main__":
 	import unittest
 

frappe/tests/test_modules.py

@@ -47,7 +47,7 @@ class TestUtils(FrappeTestCase):
 
 		if self._testMethodName == "test_export_doc":
 			self.note = frappe.new_doc("Note")
-			self.note.title = frappe.generate_hash("Note", length=10)
+			self.note.title = frappe.generate_hash(length=10)
 			self.note.save()
 
 		if self._testMethodName == "test_make_boilerplate":
@@ -69,7 +69,7 @@ class TestUtils(FrappeTestCase):
 			delattr(self, "note")
 
 		if self._testMethodName == "test_make_boilerplate":
-			self.doctype.delete()
+			self.doctype.delete(force=True)
 			scrubbed = frappe.scrub(self.doctype.name)
 			self.addCleanup(
 				delete_path,

frappe/tests/test_query.py

@@ -1,8 +1,55 @@
+import itertools
+
 import frappe
+from frappe.core.doctype.doctype.test_doctype import new_doctype
 from frappe.query_builder import Field
 from frappe.query_builder.functions import Abs, Count, Ifnull, Max, Now, Timestamp
 from frappe.tests.test_query_builder import db_type_is, run_only_if
 from frappe.tests.utils import FrappeTestCase
+from frappe.utils.nestedset import get_ancestors_of, get_descendants_of
+
+
+def create_tree_docs():
+	records = [
+		{
+			"some_fieldname": "Root Node",
+			"parent_test_tree_doctype": None,
+			"is_group": 1,
+		},
+		{
+			"some_fieldname": "Parent 1",
+			"parent_test_tree_doctype": "Root Node",
+			"is_group": 1,
+		},
+		{
+			"some_fieldname": "Parent 2",
+			"parent_test_tree_doctype": "Root Node",
+			"is_group": 1,
+		},
+		{
+			"some_fieldname": "Child 1",
+			"parent_test_tree_doctype": "Parent 1",
+			"is_group": 0,
+		},
+		{
+			"some_fieldname": "Child 2",
+			"parent_test_tree_doctype": "Parent 1",
+			"is_group": 0,
+		},
+		{
+			"some_fieldname": "Child 3",
+			"parent_test_tree_doctype": "Parent 2",
+			"is_group": 0,
+		},
+	]
+
+	tree_doctype = new_doctype("Test Tree DocType", is_tree=True, autoname="field:some_fieldname")
+	tree_doctype.insert()
+
+	for record in records:
+		d = frappe.new_doc("Test Tree DocType")
+		d.update(record)
+		d.insert()
 
 
 class TestQuery(FrappeTestCase):
@@ -17,7 +64,7 @@ class TestQuery(FrappeTestCase):
 					["DocType", "parent", "=", "something"],
 				],
 			).get_sql(),
-			"SELECT * FROM `tabDocType` LEFT JOIN `tabBOM Update Log` ON `tabBOM Update Log`.`parent`=`tabDocType`.`name` WHERE `tabBOM Update Log`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'",
+			"SELECT * FROM `tabDocType` LEFT JOIN `tabBOM Update Log` ON `tabBOM Update Log`.`parent`=`tabDocType`.`name` AND `tabBOM Update Log`.`parenttype`='DocType' WHERE `tabBOM Update Log`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'",
 		)
 
 	@run_only_if(db_type_is.MARIADB)
@@ -194,11 +241,9 @@ class TestQuery(FrappeTestCase):
 				"Note",
 				filters={"name": "Test Note Title"},
 				fields=["name", "`tabNote Seen By`.`user` as seen_by"],
-			).run(as_dict=1),
-			frappe.get_list(
-				"Note",
-				filters={"name": "Test Note Title"},
-				fields=["name", "`tabNote Seen By`.`user` as seen_by"],
+			).get_sql(),
+			"SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'".replace(
+				"`", '"' if frappe.db.db_type == "postgres" else "`"
 			),
 		)
 
@@ -207,11 +252,20 @@ class TestQuery(FrappeTestCase):
 				"Note",
 				filters={"name": "Test Note Title"},
 				fields=["name", "`tabNote Seen By`.`user` as seen_by", "`tabNote Seen By`.`idx` as idx"],
-			).run(as_dict=1),
-			frappe.get_list(
+			).get_sql(),
+			"SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'".replace(
+				"`", '"' if frappe.db.db_type == "postgres" else "`"
+			),
+		)
+
+		self.assertEqual(
+			frappe.qb.engine.get_query(
 				"Note",
 				filters={"name": "Test Note Title"},
-				fields=["name", "`tabNote Seen By`.`user` as seen_by", "`tabNote Seen By`.`idx` as idx"],
+				fields=["name", "seen_by.user as seen_by", "`tabNote Seen By`.`idx` as idx"],
+			).get_sql(),
+			"SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'".replace(
+				"`", '"' if frappe.db.db_type == "postgres" else "`"
 			),
 		)
 
@@ -220,3 +274,64 @@ class TestQuery(FrappeTestCase):
 		self.assertNotIn(
 			"email", frappe.qb.engine.get_query("User", fields=["name", "#email"], filters={}).get_sql()
 		)
+
+	def test_nestedset(self):
+		frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'")
+		frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`")
+		create_tree_docs()
+		descendants_result = frappe.qb.engine.get_query(
+			"Test Tree DocType",
+			fields=["name"],
+			filters={"name": ("descendants of", "Parent 1")},
+			orderby="modified",
+		).run(as_list=1)
+
+		# Format decendants result
+		descendants_result = list(itertools.chain.from_iterable(descendants_result))
+		self.assertListEqual(descendants_result, get_descendants_of("Test Tree DocType", "Parent 1"))
+
+		ancestors_result = frappe.qb.engine.get_query(
+			"Test Tree DocType",
+			fields=["name"],
+			filters={"name": ("ancestors of", "Child 2")},
+			orderby="modified",
+		).run(as_list=1)
+
+		# Format ancestors result
+		ancestors_result = list(itertools.chain.from_iterable(ancestors_result))
+		self.assertListEqual(ancestors_result, get_ancestors_of("Test Tree DocType", "Child 2"))
+
+		not_descendants_result = frappe.qb.engine.get_query(
+			"Test Tree DocType",
+			fields=["name"],
+			filters={"name": ("not descendants of", "Parent 1")},
+			orderby="modified",
+		).run(as_dict=1)
+
+		self.assertListEqual(
+			not_descendants_result,
+			frappe.db.get_all(
+				"Test Tree DocType",
+				fields=["name"],
+				filters={"name": ("not descendants of", "Parent 1")},
+			),
+		)
+
+		not_ancestors_result = frappe.qb.engine.get_query(
+			"Test Tree DocType",
+			fields=["name"],
+			filters={"name": ("not ancestors of", "Child 2")},
+			orderby="modified",
+		).run(as_dict=1)
+
+		self.assertListEqual(
+			not_ancestors_result,
+			frappe.db.get_all(
+				"Test Tree DocType",
+				fields=["name"],
+				filters={"name": ("not ancestors of", "Child 2")},
+			),
+		)
+
+		frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'")
+		frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`")

frappe/tests/test_query_builder.py

@@ -373,11 +373,3 @@ class TestMisc(FrappeTestCase):
 
 		DocType = Table("DocType")
 		self.assertEqual(DocType.get_sql(), "DocType")
-
-	def test_error_on_query_class(self):
-		import frappe.query_builder.utils
-
-		frappe.query_builder.utils.get_type_hints = lambda x: {"return": None}
-
-		with self.assertRaises(frappe.query_builder.utils.BuilderIdentificationFailed):
-			frappe.query_builder.utils.patch_query_execute()

frappe/tests/test_redis.py

@@ -1,4 +1,5 @@
 import functools
+from unittest.mock import patch
 
 import redis
 
@@ -30,12 +31,14 @@ def skip_if_redis_version_lt(version):
 
 class TestRedisAuth(FrappeTestCase):
 	@skip_if_redis_version_lt("6.0")
+	@patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False})
 	def test_rq_gen_acllist(self):
 		"""Make sure that ACL list is genrated"""
 		acl_list = RedisQueue.gen_acl_list()
 		self.assertEqual(acl_list[1]["bench"][0], get_bench_id())
 
 	@skip_if_redis_version_lt("6.0")
+	@patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False})
 	def test_adding_redis_user(self):
 		acl_list = RedisQueue.gen_acl_list()
 		username, password = acl_list[1]["bench"]
@@ -47,6 +50,7 @@ class TestRedisAuth(FrappeTestCase):
 		conn.acl_deluser(username)
 
 	@skip_if_redis_version_lt("6.0")
+	@patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False})
 	def test_rq_namespace(self):
 		"""Make sure that user can access only their respective namespace."""
 		# Current bench ID

frappe/tests/test_safe_exec.py

@@ -1,3 +1,5 @@
+import types
+
 import frappe
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils.safe_exec import get_safe_globals, safe_exec
@@ -59,3 +61,17 @@ class TestSafeExec(FrappeTestCase):
 
 		# enqueue whitelisted method
 		safe_exec("""frappe.enqueue("ping", now=True)""")
+
+	def test_ensure_getattrable_globals(self):
+		def check_safe(objects):
+			for obj in objects:
+				if isinstance(obj, (types.ModuleType, types.CodeType, types.TracebackType, types.FrameType)):
+					self.fail(f"{obj} wont work in safe exec.")
+				elif isinstance(obj, dict):
+					check_safe(obj.values())
+
+		check_safe(get_safe_globals().values())
+
+	def test_unsafe_objects(self):
+		unsafe_global = {"frappe": frappe}
+		self.assertRaises(SyntaxError, safe_exec, """frappe.msgprint("Hello")""", unsafe_global)

frappe/tests/test_utils.py

@@ -493,6 +493,10 @@ class TestDateUtils(FrappeTestCase):
 			frappe.utils.get_last_day_of_week("2020-12-28"), frappe.utils.getdate("2021-01-02")
 		)
 
+	def test_is_last_day_of_the_month(self):
+		self.assertEqual(frappe.utils.is_last_day_of_the_month("2020-12-24"), False)
+		self.assertEqual(frappe.utils.is_last_day_of_the_month("2020-12-31"), True)
+
 	def test_get_time(self):
 		datetime_input = now_datetime()
 		timedelta_input = get_timedelta()

frappe/tests/test_virtual_doctype.py

@@ -87,7 +87,7 @@ class TestVirtualDoctypes(FrappeTestCase):
 		cls.addClassCleanup(frappe.flags.pop, "allow_doctype_export", None)
 
 		vdt = new_doctype(name=TEST_DOCTYPE_NAME, is_virtual=1, custom=0).insert()
-		cls.addClassCleanup(vdt.delete)
+		cls.addClassCleanup(vdt.delete, force=True)
 
 		patch_virtual_doc = patch(
 			"frappe.controllers", new={frappe.local.site: {TEST_DOCTYPE_NAME: VirtualDoctypeTest}}

frappe/tests/test_zbg_job_sanity_test.py

@@ -0,0 +1,27 @@
+""" smoak tests to check that all registered background jobs execute without error.
+
+Note: Filename is intentional to run this test roughly at end. Don't change."""
+
+import time
+
+import frappe
+from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs
+from frappe.tests.utils import FrappeTestCase, timeout
+
+
+class TestScheduledJobSanity(FrappeTestCase):
+	def setUp(self):
+		remove_failed_jobs()
+
+	@timeout(90)
+	def test_bg_jobs_run(self):
+		"""Enqueue all scheduled jobs, wait for finish and verify that none failed."""
+		for scheduled_job_type in frappe.get_all("Scheduled Job Type", pluck="name"):
+			frappe.get_doc("Scheduled Job Type", scheduled_job_type).enqueue(force=True)
+
+		while RQJob.get_list({"filters": [["RQ Job", "status", "in", ("queued", "started")]]}):
+			time.sleep(0.5)
+
+		# Check no failed, if failed print full details
+		failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]})
+		self.assertEqual(len(failed_jobs), 0, "Jobs failed: " + str(failed_jobs))

frappe/tests/ui_test_helpers.py

@@ -451,7 +451,7 @@ def create_test_user():
 
 @frappe.whitelist()
 def setup_tree_doctype():
-	frappe.delete_doc_if_exists("DocType", "Custom Tree")
+	frappe.delete_doc_if_exists("DocType", "Custom Tree", force=True)
 
 	frappe.get_doc(
 		{
@@ -475,7 +475,7 @@ def setup_tree_doctype():
 
 @frappe.whitelist()
 def setup_image_doctype():
-	frappe.delete_doc_if_exists("DocType", "Custom Image")
+	frappe.delete_doc_if_exists("DocType", "Custom Image", force=True)
 
 	frappe.get_doc(
 		{

frappe/translations/de.csv

@@ -4080,7 +4080,7 @@ Scheduler Event,Scheduler-Ereignis,
 Select Event Type,Wählen Sie den Ereignistyp,
 Schedule Script,Zeitplan-Skript,
 Duration,Dauer,
-Donut,Krapfen,
+Donut,Donut,
 Custom Options,Benutzerdefinierte Optionen,
 "Ex: ""colors"": [""#d1d8dd"", ""#ff5858""]","Beispiel: "Farben": ["# d1d8dd", "# ff5858"]",
 Confirmation Email Template,Bestätigungs-E-Mail-Vorlage,
@@ -4818,3 +4818,8 @@ K,Tsd,Number system
 M,Mio,Number system
 B,Mrd,Number system
 T,Bio,Number system
+Type of Chart,Diagrammtyp,
+Preview Chart,Vorschau erzeugen,
+Please select X and Y fields,Bitte Felder für die X- und Y-Achse wählen,
+Notification sent to,Benachrichtigung gesendet an,
+Add to this activity by mailing to {0},"Senden Sie eine E-Mail an {0}, damit sie hier erscheint",

frappe/utils/background_jobs.py

@@ -3,13 +3,12 @@ import socket
 import time
 from collections import defaultdict
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Union
 from uuid import uuid4
 
 import redis
 from redis.exceptions import BusyLoadingError, ConnectionError
 from rq import Connection, Queue, Worker
-from rq.command import send_stop_job_command
 from rq.logutils import setup_loghandlers
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 
@@ -53,6 +52,8 @@ def enqueue(
 	method,
 	queue="default",
 	timeout=None,
+	on_success=None,
+	on_failure=None,
 	event=None,
 	is_async=True,
 	job_name=None,
@@ -61,7 +62,7 @@ def enqueue(
 	*,
 	at_front=False,
 	**kwargs,
-) -> "Job" | Any:
+) -> Union["Job", Any]:
 	"""
 	Enqueue method to be executed using a background worker
 
@@ -117,6 +118,8 @@ def enqueue(
 
 	return q.enqueue_call(
 		execute_job,
+		on_success=on_success,
+		on_failure=on_failure,
 		timeout=timeout,
 		kwargs=queue_args,
 		at_front=at_front,

frappe/utils/data.py

@@ -471,6 +471,12 @@ def get_last_day(dt):
 	return get_first_day(dt, 0, 1) + datetime.timedelta(-1)
 
 
+def is_last_day_of_the_month(dt):
+	last_day_of_the_month = get_last_day(dt)
+
+	return getdate(dt) == getdate(last_day_of_the_month)
+
+
 def get_quarter_ending(date):
 	date = getdate(date)
 

frappe/utils/deprecations.py

@@ -0,0 +1,27 @@
+""" Utils for deprecating functionality in Framework.
+
+WARNING: This file is internal, instead of depending just copy the code or use deprecation
+libraries.
+"""
+import functools
+import warnings
+
+
+def deprecated(func):
+	"""Decorator to wrap a function/method as deprecated."""
+
+	@functools.wraps(func)
+	def wrapper(*args, **kwargs):
+		deprecation_warning(
+			f"{func.__name__} is deprecated and will be removed in next major version.",
+			stacklevel=1,
+		)
+		return func(*args, **kwargs)
+
+	return wrapper
+
+
+def deprecation_warning(message, category=DeprecationWarning, stacklevel=1):
+	"""like warnings.warn but with auto incremented sane stacklevel."""
+
+	warnings.warn(message=message, category=category, stacklevel=stacklevel + 2)

frappe/utils/jinja_globals.py

@@ -92,9 +92,7 @@ def web_blocks(blocks):
 def get_dom_id(seed=None):
 	from frappe import generate_hash
 
-	if not seed:
-		seed = "DOM"
-	return "id-" + generate_hash(seed, 12)
+	return "id-" + generate_hash(12)
 
 
 def include_script(path, preload=True):

frappe/utils/oauth.py

@@ -266,7 +266,7 @@ def update_oauth_user(user, data, provider):
 				"email": get_email(data),
 				"gender": gender,
 				"enabled": 1,
-				"new_password": frappe.generate_hash(get_email(data)),
+				"new_password": frappe.generate_hash(),
 				"location": data.get("location"),
 				"user_type": "Website User",
 				"user_image": data.get("picture") or data.get("avatar_url"),

frappe/utils/safe_exec.py

@@ -2,6 +2,9 @@ import copy
 import inspect
 import json
 import mimetypes
+import types
+from contextlib import contextmanager
+from functools import lru_cache
 
 import RestrictedPython.Guards
 from RestrictedPython import compile_restricted, safe_globals
@@ -64,14 +67,20 @@ def safe_exec(script, _globals=None, _locals=None, restrict_commit_rollback=Fals
 		exec_globals.frappe.db.pop("rollback", None)
 		exec_globals.frappe.db.pop("add_index", None)
 
-	# execute script compiled by RestrictedPython
-	frappe.flags.in_safe_exec = True
-	exec(compile_restricted(script), exec_globals, _locals)  # pylint: disable=exec-used
-	frappe.flags.in_safe_exec = False
+	with safe_exec_flags(), patched_qb():
+		# execute script compiled by RestrictedPython
+		exec(compile_restricted(script), exec_globals, _locals)  # pylint: disable=exec-used
 
 	return exec_globals, _locals
 
 
+@contextmanager
+def safe_exec_flags():
+	frappe.flags.in_safe_exec = True
+	yield
+	frappe.flags.in_safe_exec = False
+
+
 def get_safe_globals():
 	datautils = frappe._dict()
 
@@ -258,6 +267,28 @@ def call_with_form_dict(function, kwargs):
 		frappe.local.form_dict = form_dict
 
 
+@contextmanager
+def patched_qb():
+	require_patching = isinstance(frappe.qb.terms, types.ModuleType)
+	try:
+		if require_patching:
+			_terms = frappe.qb.terms
+			frappe.qb.terms = _flatten(frappe.qb.terms)
+		yield
+	finally:
+		if require_patching:
+			frappe.qb.terms = _terms
+
+
+@lru_cache
+def _flatten(module):
+	new_mod = NamespaceDict()
+	for name, obj in inspect.getmembers(module, lambda x: not inspect.ismodule(x)):
+		if not name.startswith("_"):
+			new_mod[name] = obj
+	return new_mod
+
+
 def get_python_builtins():
 	return {
 		"abs": abs,
@@ -350,6 +381,10 @@ def _getattr(object, name, default=None):
 
 	if isinstance(name, str) and (name in UNSAFE_ATTRIBUTES):
 		raise SyntaxError(f"{name} is an unsafe attribute")
+
+	if isinstance(object, (types.ModuleType, types.CodeType, types.TracebackType, types.FrameType)):
+		raise SyntaxError(f"Reading {object} attributes is not allowed")
+
 	return RestrictedPython.Guards.safer_getattr(object, name, default=default)