Merge branch 'develop' into bg-submissions

.flake8

@@ -1,37 +1,74 @@
 [flake8]
 ignore =
+    B001,
+    B007,
+    B009,
+    B010,
+    B950,
+    E101,
+    E111,
+    E114,
+    E116,
+    E117,
     E121,
+    E122,
+    E123,
+    E124,
+    E125,
     E126,
     E127,
     E128,
+    E131,
+    E201,
+    E202,
     E203,
+    E211,
+    E221,
+    E222,
+    E223,
+    E224,
     E225,
     E226,
+    E228,
     E231,
     E241,
+    E242,
     E251,
     E261,
+    E262,
     E265,
+    E266,
+    E271,
+    E272,
+    E273,
+    E274,
+    E301,
     E302,
     E303,
     E305,
+    E306,
     E402,
     E501,
+    E502,
+    E701,
+    E702,
+    E703,
     E741,
+    F401,
+    F403,
+    F405,
+    W191,
     W291,
     W292,
     W293,
     W391,
     W503,
     W504,
-    F403,
-    B007,
-    B950,
-    W191,
-    E124, # closing bracket, irritating while writing QB code
-    E131, # continuation line unaligned for hanging indent
-    E123, # closing bracket does not match indentation of opening bracket's line
-    E101, # ensured by use of black
+    E711,
+    E129,
+    F841,
+    E713,
+    E712,
 
 max-line-length = 200
-exclude=.github/helper/semgrep_rules
+exclude=,test_*.py

.github/helper/flake8.conf

@@ -1,75 +0,0 @@
-[flake8]
-ignore =
-    B001,
-    B007,
-    B009,
-    B010,
-    B950,
-    E101,
-    E111,
-    E114,
-    E116,
-    E117,
-    E121,
-    E122,
-    E123,
-    E124,
-    E125,
-    E126,
-    E127,
-    E128,
-    E131,
-    E201,
-    E202,
-    E203,
-    E211,
-    E221,
-    E222,
-    E223,
-    E224,
-    E225,
-    E226,
-    E228,
-    E231,
-    E241,
-    E242,
-    E251,
-    E261,
-    E262,
-    E265,
-    E266,
-    E271,
-    E272,
-    E273,
-    E274,
-    E301,
-    E302,
-    E303,
-    E305,
-    E306,
-    E402,
-    E501,
-    E502,
-    E701,
-    E702,
-    E703,
-    E741,
-    F401,
-    F403,
-    F405,
-    W191,
-    W291,
-    W292,
-    W293,
-    W391,
-    W503,
-    W504,
-    E711,
-    E129,
-    F841,
-    E713,
-    E712,
-
-
-max-line-length = 200
-exclude=.github/helper/semgrep_rules,test_*.py

.github/workflows/linters.yml

@@ -79,7 +79,7 @@ jobs:
     steps:
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: '3.10'
       - uses: actions/checkout@v3
       - run: |
           pip install pip-audit

.mergify.yml

@@ -71,15 +71,6 @@ pull_request_rules:
         assignees:
           - "{{ author }}"
 
-  - name: backport to develop
-    conditions:
-      - label="backport develop"
-    actions:
-      backport:
-        branches:
-          - develop
-        assignees:
-          - "{{ author }}"
 
   - name: backport to version-13-pre-release
     conditions:

.pre-commit-config.yaml

@@ -59,7 +59,6 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies: ['flake8-bugbear',]
-        args: ['--config', '.github/helper/flake8.conf']
 
 ci:
     autoupdate_schedule: weekly

.stylelintrc

@@ -1,9 +0,0 @@
-{
-	"extends": ["stylelint-config-recommended"],
-	"plugins": ["stylelint-scss"],
-	"rules": {
-		"at-rule-no-unknown": null,
-		"scss/at-rule-no-unknown": true,
-		"no-descending-specificity": null
-	}
-}

bandit.yml

@@ -1 +0,0 @@
-skips: ['E0203', 'B605', 'B404', 'B603', 'B607']

cypress/integration/dashboard.js

@@ -0,0 +1,50 @@
+describe("Dashboard view", { scrollBehavior: false }, () => {
+	before(() => {
+		cy.login();
+		cy.visit("/app");
+	});
+
+	it("should load", () => {
+		const chart = "TODO-YEARLY-TRENDS";
+		const dashboard = "TODO-TEST-DASHBOARD"; // check slash in name intentionally.
+
+		cy.insert_doc(
+			"Dashboard Chart",
+			{
+				is_standard: 0,
+				chart_name: chart,
+				chart_type: "Count",
+				document_type: "ToDo",
+				parent_document_type: "",
+				based_on: "creation",
+				group_by_type: "Count",
+				timespan: "Last Year",
+				time_interval: "Yearly",
+				timeseries: 1,
+				type: "Line",
+				filters_json: "[]",
+			},
+			true
+		);
+
+		cy.insert_doc(
+			"Dashboard",
+			{
+				name: dashboard,
+				dashboard_name: dashboard,
+				is_standard: 0,
+				charts: [
+					{
+						chart: chart,
+					},
+				],
+			},
+			true
+		);
+
+		cy.visit(`/app/dashboard-view/${dashboard}`);
+
+		// expect chart to be loaded
+		cy.findByText(chart).should("be.visible");
+	});
+});

frappe/__init__.py

@@ -1018,19 +1018,15 @@ def get_precision(
 	return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency)
 
 
-def generate_hash(txt: str | None = None, length: int | None = None) -> str:
-	"""Generates random hash for given text + current timestamp + random string."""
-	import hashlib
-	import time
+def generate_hash(txt: str | None = None, length: int = 56) -> str:
+	"""Generate random hash using best available randomness source."""
+	import math
+	import secrets
 
-	from .utils import random_string
+	if not length:
+		length = 56
 
-	digest = hashlib.sha224(
-		((txt or "") + repr(time.time()) + repr(random_string(8))).encode()
-	).hexdigest()
-	if length:
-		digest = digest[:length]
-	return digest
+	return secrets.token_hex(math.ceil(length / 2))[:length]
 
 
 def reset_metadata_version():

frappe/app.py

@@ -86,7 +86,8 @@ def application(request: Request):
 
 		log_request(request, response)
 		process_response(response)
-		frappe.destroy()
+		if frappe.db:
+			frappe.db.close()
 
 	return response
 

frappe/client.py

@@ -333,11 +333,6 @@ def get_js(items):
 		with open(contentpath) as srcfile:
 			code = frappe.utils.cstr(srcfile.read())
 
-		if frappe.local.lang != "en":
-			messages = frappe.get_lang_dict("jsfile", contentpath)
-			messages = json.dumps(messages)
-			code += f"\n\n$.extend(frappe._messages, {messages})"
-
 		out.append(code)
 
 	return out

frappe/core/doctype/data_export/exporter.py

@@ -432,7 +432,7 @@ class DataExporter:
 				row[_column_start_end.start + i + 1] = value
 
 	def build_response_as_excel(self):
-		filename = frappe.generate_hash("", 10)
+		filename = frappe.generate_hash(length=10)
 		with open(filename, "wb") as f:
 			f.write(cstr(self.writer.getvalue()).encode("utf-8"))
 		f = open(filename)

frappe/core/doctype/data_import/test_importer.py

@@ -97,7 +97,7 @@ class TestImporter(FrappeTestCase):
 	def test_data_import_update(self):
 		existing_doc = frappe.get_doc(
 			doctype=doctype_name,
-			title=frappe.generate_hash(doctype_name, 8),
+			title=frappe.generate_hash(length=8),
 			table_field_1=[{"child_title": "child title to update"}],
 		)
 		existing_doc.save()

frappe/core/doctype/doctype/doctype.py

@@ -329,7 +329,7 @@ class DocType(Document):
 			"DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name)
 		)
 		for p in parent_list:
-			frappe.db.update("DocType", p.parent, {}, for_update=False)
+			frappe.db.set_value("DocType", p.parent, {}, for_update=False)
 
 	def scrub_field_names(self):
 		"""Sluggify fieldnames if not set from Label."""

frappe/core/doctype/doctype/test_doctype.py

@@ -670,6 +670,9 @@ class TestDocType(FrappeTestCase):
 
 		self.assertEqual(test_json.test_json_field["hello"], "world")
 
+	def test_no_delete_doc(self):
+		self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address")
+
 	@patch.dict(frappe.conf, {"developer_mode": 1})
 	def test_custom_field_deletion(self):
 		"""Custom child tables whose doctype doesn't exist should be auto deleted."""

frappe/core/doctype/rq_job/rq_job.py

@@ -5,10 +5,12 @@ import functools
 import re
 
 from rq.command import send_stop_job_command
+from rq.exceptions import InvalidJobOperation
 from rq.job import Job
 from rq.queue import Queue
 
 import frappe
+from frappe import _
 from frappe.model.document import Document
 from frappe.utils import (
 	cint,
@@ -93,7 +95,10 @@ class RQJob(Document):
 
 	@check_permissions
 	def stop_job(self):
-		send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id)
+		try:
+			send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id)
+		except InvalidJobOperation:
+			frappe.msgprint(_("Job is not running."), title=_("Invalid Operation"))
 
 	@staticmethod
 	def get_count(args) -> int:

frappe/core/doctype/rq_job/test_rq_job.py

@@ -19,12 +19,11 @@ class TestRQJob(FrappeTestCase):
 
 	@timeout(seconds=20)
 	def check_status(self, job: Job, status, wait=True):
-		if wait:
-			while True:
-				if job.is_queued or job.is_started:
-					time.sleep(0.2)
-				else:
-					break
+		while wait:
+			if not (job.is_queued or job.is_started):
+				break
+			time.sleep(0.2)
+
 		self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status)
 
 	def test_serialization(self):
@@ -69,7 +68,7 @@ class TestRQJob(FrappeTestCase):
 		self.assertGreaterEqual(len(non_failed_jobs), 1)
 
 		# Create a slow job and check if it's stuck in "Started"
-		job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=1000)
+		job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=10)
 		time.sleep(3)
 		self.check_status(job, "started", wait=False)
 		stop_job(job_id=job.id)
@@ -84,8 +83,8 @@ class TestRQJob(FrappeTestCase):
 
 	def test_is_enqueued(self):
 
+		dummy_job = frappe.enqueue(self.BG_JOB, sleep=10, queue="short")
 		job_name = "uniq_test_job"
-		dummy_job = frappe.enqueue(self.BG_JOB, sleep=100, queue="short")
 		actual_job = frappe.enqueue(self.BG_JOB, job_name=job_name, queue="short")
 
 		self.assertTrue(is_job_queued(job_name))

frappe/core/doctype/server_script/test_server_script.py

@@ -211,3 +211,25 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
 """
 		script.save()
 		script.execute_method()
+
+	def test_scripts_all_the_way_down(self):
+		# why not
+		script = frappe.get_doc(
+			doctype="Server Script",
+			name="test_nested_scripts_1",
+			script_type="API",
+			api_method="test_nested_scripts_1",
+			script=f"""log("nothing")""",
+		)
+		script.insert()
+		script.execute_method()
+
+		script = frappe.get_doc(
+			doctype="Server Script",
+			name="test_nested_scripts_2",
+			script_type="API",
+			api_method="test_nested_scripts_2",
+			script=f"""frappe.call("test_nested_scripts_1")""",
+		)
+		script.insert()
+		script.execute_method()

frappe/core/doctype/user/user.py

@@ -471,7 +471,7 @@ class User(Document):
 			frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False)
 
 		# set email
-		frappe.db.update("User", new_name, "email", new_name)
+		frappe.db.set_value("User", new_name, "email", new_name)
 
 	def append_roles(self, *roles):
 		"""Add roles to user"""

frappe/core/page/permission_manager/permission_manager.py

@@ -19,7 +19,6 @@ from frappe.permissions import (
 	setup_custom_perms,
 	update_permission_property,
 )
-from frappe.translate import send_translations
 from frappe.utils.user import get_users_with_role as _get_user_with_role
 
 not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"]
@@ -28,7 +27,6 @@ not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Tran
 @frappe.whitelist()
 def get_roles_and_doctypes():
 	frappe.only_for("System Manager")
-	send_translations(frappe.get_lang_dict("doctype", "DocPerm"))
 
 	active_domains = frappe.get_active_domains()
 

frappe/database/database.py

@@ -7,7 +7,7 @@ import random
 import re
 import string
 import traceback
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from time import time
 
 from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
@@ -29,7 +29,8 @@ from frappe.exceptions import DoesNotExistError, ImplicitCommitError
 from frappe.model.utils.link_count import flush_local_link_count
 from frappe.query_builder.functions import Count
 from frappe.utils import cast as cast_fieldtype
-from frappe.utils import get_datetime, get_table_name, getdate, now, sbool
+from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool
+from frappe.utils.deprecations import deprecated
 
 IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE)
 INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*")
@@ -114,6 +115,17 @@ class Database:
 		self._cursor = self._conn.cursor()
 		frappe.local.rollback_observers = []
 
+		try:
+			if execution_timeout := get_query_execution_timeout():
+				self.set_execution_timeout(execution_timeout)
+		except Exception as e:
+			frappe.logger("database").warning(f"Couldn't set execution timeout {e}")
+
+	def set_execution_timeout(self, seconds: int):
+		"""Set session speicifc timeout on exeuction of statements.
+		If any statement takes more time it will be killed along with entire transaction."""
+		raise NotImplementedError
+
 	def use(self, db_name):
 		"""`USE` db_name."""
 		self._conn.select_db(db_name)
@@ -135,12 +147,11 @@ class Database:
 		self,
 		query: Query,
 		values: QueryValues = EmptyQueryValues,
+		*,
 		as_dict=0,
 		as_list=0,
-		formatted=0,
 		debug=0,
 		ignore_ddl=0,
-		as_utf8=0,
 		auto_commit=0,
 		update=None,
 		explain=False,
@@ -153,10 +164,8 @@ class Database:
 		:param values: Tuple / List / Dict of values to be escaped and substituted in the query.
 		:param as_dict: Return as a dictionary.
 		:param as_list: Always return as a list.
-		:param formatted: Format values like date etc.
 		:param debug: Print query and `EXPLAIN` in debug log.
 		:param ignore_ddl: Catch exception if table, column missing.
-		:param as_utf8: Encode values as UTF 8.
 		:param auto_commit: Commit after executing the query.
 		:param update: Update this dict to all rows (if returned `as_dict`).
 		:param run: Returns query without executing it if False.
@@ -264,13 +273,13 @@ class Database:
 
 		# scrub output if required
 		if as_dict:
-			ret = self.fetch_as_dict(formatted, as_utf8)
+			ret = self.fetch_as_dict()
 			if update:
 				for r in ret:
 					r.update(update)
 			return ret
-		elif as_list or as_utf8:
-			return self.convert_to_lists(self.last_result, formatted, as_utf8)
+		elif as_list:
+			return self.convert_to_lists(self.last_result)
 		return self.last_result
 
 	def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None:
@@ -377,56 +386,27 @@ class Database:
 		):
 			raise ImplicitCommitError("This statement can cause implicit commit")
 
-	def fetch_as_dict(self, formatted=0, as_utf8=0) -> list[frappe._dict]:
+	def fetch_as_dict(self) -> list[frappe._dict]:
 		"""Internal. Converts results to dict."""
 		result = self.last_result
-		ret = []
 		if result:
 			keys = [column[0] for column in self._cursor.description]
 
-		for r in result:
-			values = []
-			for value in r:
-				if as_utf8 and isinstance(value, str):
-					value = value.encode("utf-8")
-				values.append(value)
-
-			ret.append(frappe._dict(zip(keys, values)))
-		return ret
+		return [frappe._dict(zip(keys, row)) for row in result]
 
 	@staticmethod
 	def clear_db_table_cache(query):
 		if query and is_query_type(query, ("drop", "create")):
 			frappe.cache().delete_key("db_tables")
 
-	@staticmethod
-	def needs_formatting(result, formatted):
-		"""Returns true if the first row in the result has a Date, Datetime, Long Int."""
-		if result and result[0]:
-			for v in result[0]:
-				if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)):
-					return True
-				if formatted and isinstance(v, (int, float)):
-					return True
-
-		return False
-
 	def get_description(self):
 		"""Returns result metadata."""
 		return self._cursor.description
 
 	@staticmethod
-	def convert_to_lists(res, formatted=0, as_utf8=0):
+	def convert_to_lists(res):
 		"""Convert tuple output to lists (internal)."""
-		nres = []
-		for r in res:
-			nr = []
-			for val in r:
-				if as_utf8 and isinstance(val, str):
-					val = val.encode("utf-8")
-				nr.append(val)
-			nres.append(nr)
-		return nres
+		return [[value for value in row] for row in res]
 
 	def get(self, doctype, filters=None, as_dict=True, cache=False):
 		"""Returns `get_value` with fieldname='*'"""
@@ -826,10 +806,6 @@ class Database:
 			).run(debug=debug, run=run, as_dict=as_dict)
 		return {}
 
-	def update(self, *args, **kwargs):
-		"""Update multiple values. Alias for `set_value`."""
-		return self.set_value(*args, **kwargs)
-
 	def set_value(
 		self,
 		dt,
@@ -855,7 +831,6 @@ class Database:
 		:param modified_by: Set this user as `modified_by`.
 		:param update_modified: default True. Set as false, if you don't want to update the timestamp.
 		:param debug: Print the query in the developer / js console.
-		:param for_update: [DEPRECATED] This function now performs updates in single query, locking is not required.
 		"""
 		is_single_doctype = not (dn and dt != dn)
 		to_update = field if isinstance(field, dict) else {field: val}
@@ -895,30 +870,6 @@ class Database:
 		if dt in self.value_cache:
 			del self.value_cache[dt]
 
-	@staticmethod
-	def set(doc, field, val):
-		"""Set value in document. **Avoid**"""
-		doc.db_set(field, val)
-
-	def touch(self, doctype, docname):
-		"""Update the modified timestamp of this document."""
-		modified = now()
-		DocType = frappe.qb.DocType(doctype)
-		frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run()
-		return modified
-
-	@staticmethod
-	def set_temp(value):
-		"""Set a temperory value and return a key."""
-		key = frappe.generate_hash()
-		frappe.cache().hset("temp", key, value)
-		return key
-
-	@staticmethod
-	def get_temp(key):
-		"""Return the temperory value and delete it."""
-		return frappe.cache().hget("temp", key)
-
 	def set_global(self, key, val, user="__global"):
 		"""Save a global key value. Global values will be automatically set if they match fieldname."""
 		self.set_default(key, val, user)
@@ -1078,7 +1029,7 @@ class Database:
 		return getdate(date).strftime("%Y-%m-%d")
 
 	@staticmethod
-	def format_datetime(datetime):
+	def format_datetime(datetime):  # noqa: F811
 		if not datetime:
 			return FallBackDateTimeStr
 
@@ -1222,9 +1173,6 @@ class Database:
 		"""
 		return self.sql_ddl(f"truncate `{get_table_name(doctype)}`")
 
-	def clear_table(self, doctype):
-		return self.truncate(doctype)
-
 	def get_last_created(self, doctype):
 		last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc")
 		if last_record:
@@ -1340,3 +1288,28 @@ def savepoint(catch: type | tuple[type, ...] = Exception):
 		frappe.db.rollback(save_point=savepoint)
 	else:
 		frappe.db.release_savepoint(savepoint)
+
+
+def get_query_execution_timeout() -> int:
+	"""Get execution timeout based on current timeout in different contexts.
+
+	    HTTP requests: HTTP timeout or a default (300)
+	    Background jobs: Job timeout
+	Console/Commands: No timeout = 0.
+
+	    Note: Timeout adds 1.5x as "safety factor"
+	"""
+	from rq import get_current_job
+
+	if not frappe.conf.get("enable_db_statement_timeout"):
+		return 0
+
+	# Zero means no timeout, which is the default value in db.
+	timeout = 0
+	with suppress(Exception):
+		if getattr(frappe.local, "request", None):
+			timeout = frappe.conf.http_timeout or 300
+		elif job := get_current_job():
+			timeout = job.timeout
+
+	return int(cint(timeout) * 1.5)

frappe/database/mariadb/database.py

@@ -68,6 +68,10 @@ class MariaDBExceptionUtil:
 	def is_syntax_error(e: pymysql.Error) -> bool:
 		return e.args[0] == ER.PARSE_ERROR
 
+	@staticmethod
+	def is_statement_timeout(e: pymysql.Error) -> bool:
+		return e.args[0] == 1969
+
 	@staticmethod
 	def is_data_too_long(e: pymysql.Error) -> bool:
 		return e.args[0] == ER.DATA_TOO_LONG
@@ -102,6 +106,9 @@ class MariaDBConnectionUtil:
 	def create_connection(self):
 		return pymysql.connect(**self.get_connection_settings())
 
+	def set_execution_timeout(self, seconds: int):
+		self.sql("set session max_statement_time = %s", int(seconds))
+
 	def get_connection_settings(self) -> dict:
 		conn_settings = {
 			"host": self.host,

frappe/database/postgres/database.py

@@ -99,6 +99,10 @@ class PostgresExceptionUtil:
 	def is_duplicate_fieldname(e):
 		return getattr(e, "pgcode", None) == DUPLICATE_COLUMN
 
+	@staticmethod
+	def is_statement_timeout(e):
+		return PostgresDatabase.is_timedout(e) or isinstance(e, frappe.QueryTimeoutError)
+
 	@staticmethod
 	def is_data_too_long(e):
 		return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION
@@ -161,6 +165,10 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 
 		return conn
 
+	def set_execution_timeout(self, seconds: int):
+		# Postgres expects milliseconds as input
+		self.sql("set local statement_timeout = %s", int(seconds) * 1000)
+
 	def escape(self, s, percent=True):
 		"""Escape quotes and percent in given string."""
 		if isinstance(s, bytes):

frappe/desk/desk_page.py

@@ -2,7 +2,6 @@
 # License: MIT. See LICENSE
 
 import frappe
-from frappe.translate import send_translations
 
 
 @frappe.whitelist()
@@ -31,10 +30,6 @@ def getpage():
 	page = frappe.form_dict.get("name")
 	doc = get(page)
 
-	# load translations
-	if frappe.lang != "en":
-		send_translations(frappe.get_lang_dict("page", page))
-
 	frappe.response.docs.append(doc)
 
 

frappe/desk/query_report.py

@@ -14,7 +14,6 @@ from frappe.model.utils import render_include
 from frappe.modules import get_module_path, scrub
 from frappe.monitor import add_data_to_monitor
 from frappe.permissions import get_role_permissions
-from frappe.translate import send_translations
 from frappe.utils import (
 	cint,
 	cstr,
@@ -202,10 +201,6 @@ def get_script(report_name):
 	if not script:
 		script = "frappe.query_reports['%s']={}" % report_name
 
-	# load translations
-	if frappe.lang != "en":
-		send_translations(frappe.get_lang_dict("report", report_name))
-
 	return {
 		"script": render_include(script),
 		"html_format": html_format,

frappe/desk/reportview.py

@@ -91,7 +91,7 @@ def validate_args(data):
 def validate_fields(data):
 	wildcard = update_wildcard_field_param(data)
 
-	for field in data.fields or []:
+	for field in list(data.fields or []):
 		fieldname = extract_fieldname(field)
 		if is_standard(fieldname):
 			continue

frappe/email/doctype/newsletter/test_newsletter.py

@@ -74,7 +74,7 @@ class TestNewsletterMixin:
 				).insert(ignore_if_duplicate=True)
 			except Exception:
 				frappe.db.rollback(save_point=savepoint)
-				frappe.db.update(doctype, email_filters, "unsubscribed", 0)
+				frappe.db.set_value(doctype, email_filters, "unsubscribed", 0)
 
 			frappe.db.release_savepoint(savepoint)
 

frappe/installer.py

@@ -402,7 +402,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
 
 			if not dry_run:
 				if doctype.issingle:
-					frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True)
+					frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True, force=True)
 				else:
 					drop_doctypes.append(doctype.name)
 
@@ -460,7 +460,7 @@ def _delete_doctypes(doctypes: list[str], dry_run: bool) -> None:
 	for doctype in set(doctypes):
 		print(f"* dropping Table for '{doctype}'...")
 		if not dry_run:
-			frappe.delete_doc("DocType", doctype, ignore_on_trash=True)
+			frappe.delete_doc("DocType", doctype, ignore_on_trash=True, force=True)
 			frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{doctype}`")
 
 

frappe/model/delete_doc.py

@@ -92,6 +92,8 @@ def delete_doc(
 
 			else:
 				doc = frappe.get_doc(doctype, name)
+				if not (doc.custom or frappe.conf.developer_mode or frappe.flags.in_patch or force):
+					frappe.throw(_("Standard DocType can not be deleted."))
 
 				update_flags(doc, flags, ignore_permissions)
 				check_permission_and_not_submitted(doc)

frappe/model/document.py

@@ -146,9 +146,14 @@ class Document(BaseDocument):
 			self._fix_numeric_types()
 
 		else:
+			get_value_kwargs = {"for_update": self.flags.for_update, "as_dict": True}
+			if not isinstance(self.name, (dict, list)):
+				get_value_kwargs["order_by"] = None
+
 			d = frappe.db.get_value(
-				self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update
+				doctype=self.doctype, filters=self.name, fieldname="*", **get_value_kwargs
 			)
+
 			if not d:
 				frappe.throw(
 					_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError
@@ -753,12 +758,13 @@ class Document(BaseDocument):
 		Will also validate document transitions (Save > Submit > Cancel) calling
 		`self.check_docstatus_transition`."""
 
-		self.load_doc_before_save()
+		self.load_doc_before_save(raise_exception=True)
 
 		self._action = "save"
-		previous = self.get_doc_before_save()
+		previous = self._doc_before_save
 
-		if not previous or self.meta.get("is_virtual"):
+		# previous is None for new document insert
+		if not previous:
 			self.check_docstatus_transition(0)
 			return
 
@@ -1057,7 +1063,7 @@ class Document(BaseDocument):
 
 		self.set_title_field()
 
-	def load_doc_before_save(self):
+	def load_doc_before_save(self, *, raise_exception: bool = False):
 		"""load existing document from db before saving"""
 
 		self._doc_before_save = None
@@ -1068,6 +1074,9 @@ class Document(BaseDocument):
 		try:
 			self._doc_before_save = frappe.get_doc(self.doctype, self.name, for_update=True)
 		except frappe.DoesNotExistError:
+			if raise_exception:
+				raise
+
 			frappe.clear_last_message()
 
 	def run_post_save_methods(self):

frappe/model/naming.py

@@ -278,7 +278,7 @@ def make_autoname(key="", doctype="", doc=""):
 	                DE/09/01/00001 where 09 is the year, 01 is the month and 00001 is the series
 	"""
 	if key == "hash":
-		return frappe.generate_hash(doctype, 10)
+		return frappe.generate_hash(length=10)
 
 	series = NamingSeries(key)
 	return series.generate_next_name(doc)

frappe/model/workflow.py

@@ -1,12 +1,17 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import json
+from typing import TYPE_CHECKING, Union
 
 import frappe
 from frappe import _
 from frappe.model.docstatus import DocStatus
 from frappe.utils import cint
 
+if TYPE_CHECKING:
+	from frappe.model.document import Document
+	from frappe.workflow.doctype.workflow.workflow import Workflow
+
 
 class WorkflowStateError(frappe.ValidationError):
 	pass
@@ -32,20 +37,22 @@ def get_workflow_name(doctype):
 
 
 @frappe.whitelist()
-def get_transitions(doc, workflow=None, raise_exception=False):
+def get_transitions(
+	doc: Union["Document", str, dict], workflow: "Workflow" = None, raise_exception: bool = False
+) -> list[dict]:
 	"""Return list of possible transitions for the given doc"""
-	doc = frappe.get_doc(frappe.parse_json(doc))
+	from frappe.model.document import Document
+
+	if not isinstance(doc, Document):
+		doc = frappe.get_doc(frappe.parse_json(doc))
+		doc.load_from_db()
 
 	if doc.is_new():
 		return []
 
-	doc.load_from_db()
+	doc.check_permission("read")
 
-	frappe.has_permission(doc, "read", throw=True)
-	roles = frappe.get_roles()
-
-	if not workflow:
-		workflow = get_workflow(doc.doctype)
+	workflow = workflow or get_workflow(doc.doctype)
 	current_state = doc.get(workflow.workflow_state_field)
 
 	if not current_state:
@@ -55,11 +62,14 @@ def get_transitions(doc, workflow=None, raise_exception=False):
 			frappe.throw(_("Workflow State not set"), WorkflowStateError)
 
 	transitions = []
+	roles = frappe.get_roles()
+
 	for transition in workflow.transitions:
 		if transition.state == current_state and transition.allowed in roles:
 			if not is_transition_condition_satisfied(transition, doc):
 				continue
 			transitions.append(transition.as_dict())
+
 	return transitions
 
 
@@ -79,7 +89,7 @@ def get_workflow_safe_globals():
 	)
 
 
-def is_transition_condition_satisfied(transition, doc):
+def is_transition_condition_satisfied(transition, doc) -> bool:
 	if not transition.condition:
 		return True
 	else:
@@ -198,7 +208,7 @@ def validate_workflow(doc):
 			)
 
 
-def get_workflow(doctype):
+def get_workflow(doctype) -> "Workflow":
 	return frappe.get_doc("Workflow", get_workflow_name(doctype))
 
 

frappe/patches/v11_0/remove_skip_for_doctype.py

@@ -60,7 +60,7 @@ def execute():
 					# Maintain sequence (name, user, allow, for_value, applicable_for, apply_to_all_doctypes, creation, modified)
 					new_user_permissions_list.append(
 						(
-							frappe.generate_hash("", 10),
+							frappe.generate_hash(length=10),
 							user_permission.user,
 							user_permission.allow,
 							user_permission.for_value,

frappe/patches/v12_0/move_email_and_phone_to_child_table.py

@@ -27,7 +27,7 @@ def execute():
 			email_values.append(
 				(
 					1,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.email_id,
 					"email_ids",
 					"Contact",
@@ -44,7 +44,7 @@ def execute():
 			phone_values.append(
 				(
 					phone_counter,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.phone,
 					"phone_nos",
 					"Contact",
@@ -63,7 +63,7 @@ def execute():
 			phone_values.append(
 				(
 					phone_counter,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.mobile_no,
 					"phone_nos",
 					"Contact",

frappe/patches/v12_0/setup_tags.py

@@ -28,7 +28,7 @@ def execute():
 
 				tag_list.append((tag.strip(), time, time, "Administrator"))
 
-				tag_link_name = frappe.generate_hash(_user_tags.name + tag.strip() + doctype.name, 10)
+				tag_link_name = frappe.generate_hash(length=10)
 				tag_links.append(
 					(tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, "Administrator")
 				)

frappe/patches/v13_0/queryreport_columns.py

@@ -15,4 +15,4 @@ def execute():
 		if isinstance(data, list):
 			# double escape braces
 			jstr = f'{{"columns":{jstr}}}'
-			frappe.db.update("Report", record["name"], "json", jstr)
+			frappe.db.set_value("Report", record["name"], "json", jstr)

frappe/patches/v14_0/remove_db_aggregation.py

@@ -32,4 +32,4 @@ def execute():
 		for agg in ["avg", "max", "min", "sum"]:
 			script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script)
 
-		frappe.db.update("Server Script", name, "script", script)
+		frappe.db.set_value("Server Script", name, "script", script)

frappe/public/js/frappe/list/list_view.js

@@ -1179,7 +1179,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 		this.$result.on("click", ".list-row, .image-view-header, .file-header", (e) => {
 			const $target = $(e.target);
 			// tick checkbox if Ctrl/Meta key is pressed
-			if (e.ctrlKey || (e.metaKey && !$target.is("a"))) {
+			if ((e.ctrlKey || e.metaKey) && !$target.is("a")) {
 				const $list_row = $(e.currentTarget);
 				const $check = $list_row.find(".list-row-checkbox");
 				$check.prop("checked", !$check.prop("checked"));

frappe/public/js/frappe/list/list_view_select.js

@@ -315,7 +315,7 @@ frappe.views.ListViewSelect = class ListViewSelect {
 		accounts.forEach((account) => {
 			let email_account =
 				account.email_id == "All Accounts" ? "All Accounts" : account.email_account;
-			let route = `/app/communication/inbox/${email_account}`;
+			let route = `/app/communication/view/inbox/${email_account}`;
 			let display_name = ["All Accounts", "Sent Mail", "Spam", "Trash"].includes(
 				account.email_id
 			)

frappe/public/js/frappe/ui/keyboard.js

@@ -331,7 +331,7 @@ function close_grid_and_dialog() {
 	}
 
 	// close open dialog
-	if (cur_dialog && !cur_dialog.no_cancel_flag) {
+	if (cur_dialog && !cur_dialog.no_cancel_flag && !cur_dialog.static) {
 		cur_dialog.cancel();
 		return false;
 	}

frappe/public/js/frappe/ui/page.js

@@ -327,13 +327,14 @@ frappe.ui.Page = class Page {
 
 	//--- Menu --//
 
-	add_menu_item(label, click, standard, shortcut) {
+	add_menu_item(label, click, standard, shortcut, show_parent) {
 		return this.add_dropdown_item({
 			label,
 			click,
 			standard,
 			parent: this.menu,
 			shortcut,
+			show_parent,
 		});
 	}
 
@@ -424,7 +425,7 @@ frappe.ui.Page = class Page {
 		icon = null,
 	}) {
 		if (show_parent) {
-			parent.parent().removeClass("hide");
+			parent.parent().removeClass("hide hidden-xl");
 		}
 
 		let $link = this.is_in_group_button_dropdown(parent, "li > a.grey-link > span", label);
@@ -602,8 +603,11 @@ frappe.ui.Page = class Page {
 		};
 		// Add actions as menu item in Mobile View
 		let menu_item_label = group ? `${group} > ${label}` : label;
-		let menu_item = this.add_menu_item(menu_item_label, _action, false);
+		let menu_item = this.add_menu_item(menu_item_label, _action, false, false, false);
 		menu_item.parent().addClass("hidden-xl");
+		if (this.menu_btn_group.hasClass("hide")) {
+			this.menu_btn_group.removeClass("hide").addClass("hidden-xl");
+		}
 
 		if (group) {
 			var $group = this.get_or_add_inner_group_button(group);

frappe/public/js/frappe/ui/toolbar/about.js

@@ -19,6 +19,8 @@ frappe.ui.misc.about = function () {
 						Facebook: <a href='https://facebook.com/erpnext' target='_blank'>https://facebook.com/erpnext</a></p>
 					<p><i class='fa fa-twitter fa-fw'></i>
 						Twitter: <a href='https://twitter.com/erpnext' target='_blank'>https://twitter.com/erpnext</a></p>
+					<p><i class='fa fa-youtube fa-fw'></i>
+						YouTube: <a href='https://www.youtube.com/@erpnextofficial' target='_blank'>https://www.youtube.com/@erpnextofficial</a></p>
 					<hr>
 					<h4>${__("Installed Apps")}</h4>
 					<div id='about-app-versions'>${__("Loading versions...")}</div>

frappe/public/js/frappe/utils/utils.js

@@ -1383,7 +1383,7 @@ Object.assign(frappe.utils, {
 			: "";
 
 		return $(`<div class="summary-item">
-			<span class="summary-label">${summary.label}</span>
+			<span class="summary-label">${__(summary.label)}</span>
 			<div class="summary-value ${color}">${value}</div>
 		</div>`);
 	},

frappe/public/js/frappe/views/breadcrumbs.js

@@ -40,7 +40,6 @@ frappe.breadcrumbs = {
 				type: type,
 			};
 		}
-
 		this.all[frappe.breadcrumbs.current_page()] = obj;
 		this.update();
 	},
@@ -137,13 +136,13 @@ frappe.breadcrumbs = {
 		const doctype_meta = frappe.get_doc("DocType", doctype);
 		if (
 			(doctype === "User" && !frappe.user.has_role("System Manager")) ||
-			(doctype_meta && doctype_meta.issingle)
+			doctype_meta?.issingle
 		) {
 			// no user listview for non-system managers and single doctypes
 		} else {
 			let route;
 			const doctype_route = frappe.router.slug(frappe.router.doctype_layout || doctype);
-			if (doctype_meta.is_tree) {
+			if (doctype_meta?.is_tree) {
 				let view = frappe.model.user_settings[doctype].last_view || "Tree";
 				route = `${doctype_route}/view/${view}`;
 			} else {

frappe/public/js/frappe/views/reports/query_report.js

@@ -1067,7 +1067,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				{
 					fieldname: "sb_1",
 					fieldtype: "Section Break",
-					label: "Y axis",
+					label: "Y Axis",
 				},
 				{
 					fieldname: "y_axis_fields",

frappe/templates/includes/navbar/navbar_items.html

@@ -3,7 +3,7 @@
 
 {% if parent %}
 
-{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%}
+{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
 <li class="nav-item dropdown {% if submenu %} dropdown-submenu {% endif %}">
 	<a class="nav-link dropdown-toggle" href="#" id="{{ dropdown_id }}" role="button"
 		data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
@@ -16,7 +16,7 @@
 	</ul>
 </li>
 {% else %}
-{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%}
+{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
 <li class="dropdown {% if submenu %} dropdown-submenu {% endif %}">
 	<a class="dropdown-item dropdown-toggle" href="#" id="{{ dropdown_id }}" role="button"
 		data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">

frappe/templates/includes/navbar/navbar_search.html

@@ -2,8 +2,8 @@
 	<li>
 		<form action='/search'>
 		<input name='q' class='form-control navbar-search' type='text'
-			value='{{ frappe.form_dict.q or ''}}'
+			value='{{ frappe.form_dict.q|e or ''}}'
 			{% if not frappe.form_dict.q%}placeholder="{{ _("Search...") }}"{% endif %}>
 		</form>
 	</li>
-{% endif %}
+{% endif %}

frappe/test_runner.py

@@ -86,7 +86,6 @@ def main(
 			frappe.utils.scheduler.disable_scheduler()
 
 		set_test_email_config()
-		frappe.conf.update({"bench_id": "test_bench", "use_rq_auth": False})
 
 		if not frappe.flags.skip_before_tests:
 			if verbose:

frappe/tests/test_db.py

@@ -11,13 +11,13 @@ import frappe
 from frappe.core.utils import find
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.database import savepoint
-from frappe.database.database import Database
+from frappe.database.database import Database, get_query_execution_timeout
 from frappe.database.utils import FallBackDateTimeStr
 from frappe.query_builder import Field
 from frappe.query_builder.functions import Concat_ws
 from frappe.tests.test_query_builder import db_type_is, run_only_if
 from frappe.tests.utils import FrappeTestCase
-from frappe.utils import add_days, cint, now, random_string
+from frappe.utils import add_days, cint, now, random_string, set_request
 from frappe.utils.testutils import clear_custom_fields
 
 
@@ -36,6 +36,34 @@ class TestDB(FrappeTestCase):
 	def test_get_database_size(self):
 		self.assertIsInstance(frappe.db.get_database_size(), (float, int))
 
+	def test_db_statement_execution_timeout(self):
+		frappe.db.set_execution_timeout(2)
+		# Setting 0 means no timeout.
+		self.addCleanup(frappe.db.set_execution_timeout, 0)
+
+		try:
+			savepoint = "statement_timeout"
+			frappe.db.savepoint(savepoint)
+			frappe.db.multisql(
+				{
+					"mariadb": "select sleep(10)",
+					"postgres": "select pg_sleep(10)",
+				}
+			)
+		except Exception as e:
+			self.assertTrue(frappe.db.is_statement_timeout(e), f"exepcted {e} to be timeout error")
+			frappe.db.rollback(save_point=savepoint)
+		else:
+			frappe.db.rollback(save_point=savepoint)
+			self.fail("Long running queries not timing out")
+
+	@patch.dict(frappe.conf, {"http_timeout": 20, "enable_db_statement_timeout": 1})
+	def test_db_timeout_computation(self):
+		set_request(method="GET", path="/")
+		self.assertEqual(get_query_execution_timeout(), 30)
+		frappe.local.request = None
+		self.assertEqual(get_query_execution_timeout(), 0)
+
 	def test_get_value(self):
 		self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator")
@@ -763,23 +791,6 @@ class TestDBSetValue(FrappeTestCase):
 		cached_doc = frappe.get_cached_doc(self.todo2.doctype, self.todo2.name)
 		self.assertEqual(cached_doc.description, description)
 
-	def test_update_alias(self):
-		args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`")
-		kwargs = {
-			"for_update": False,
-			"modified": None,
-			"modified_by": None,
-			"update_modified": True,
-			"debug": False,
-		}
-
-		self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update))
-
-		with patch.object(Database, "set_value") as set_value:
-			frappe.db.update(*args, **kwargs)
-			set_value.assert_called_once()
-			set_value.assert_called_with(*args, **kwargs)
-
 	@classmethod
 	def tearDownClass(cls):
 		frappe.db.rollback()

frappe/tests/test_document.py

@@ -411,6 +411,19 @@ class TestDocument(FrappeTestCase):
 		todo.save()
 		self.assertEqual(todo.notify_update.call_count, 1)
 
+	def test_error_on_saving_new_doc_with_name(self):
+		"""Trying to save a new doc with name should raise DoesNotExistError"""
+
+		doc = frappe.get_doc(
+			{
+				"doctype": "ToDo",
+				"description": "this should raise frappe.DoesNotExistError",
+				"name": "lets-trick-doc-save",
+			}
+		)
+
+		self.assertRaises(frappe.DoesNotExistError, doc.save)
+
 
 class TestDocumentWebView(FrappeTestCase):
 	def get(self, path, user="Guest"):

frappe/tests/test_modules.py

@@ -47,7 +47,7 @@ class TestUtils(FrappeTestCase):
 
 		if self._testMethodName == "test_export_doc":
 			self.note = frappe.new_doc("Note")
-			self.note.title = frappe.generate_hash("Note", length=10)
+			self.note.title = frappe.generate_hash(length=10)
 			self.note.save()
 
 		if self._testMethodName == "test_make_boilerplate":
@@ -69,7 +69,7 @@ class TestUtils(FrappeTestCase):
 			delattr(self, "note")
 
 		if self._testMethodName == "test_make_boilerplate":
-			self.doctype.delete()
+			self.doctype.delete(force=True)
 			scrubbed = frappe.scrub(self.doctype.name)
 			self.addCleanup(
 				delete_path,

frappe/tests/test_redis.py

@@ -1,4 +1,5 @@
 import functools
+from unittest.mock import patch
 
 import redis
 
@@ -30,12 +31,14 @@ def skip_if_redis_version_lt(version):
 
 class TestRedisAuth(FrappeTestCase):
 	@skip_if_redis_version_lt("6.0")
+	@patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False})
 	def test_rq_gen_acllist(self):
 		"""Make sure that ACL list is genrated"""
 		acl_list = RedisQueue.gen_acl_list()
 		self.assertEqual(acl_list[1]["bench"][0], get_bench_id())
 
 	@skip_if_redis_version_lt("6.0")
+	@patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False})
 	def test_adding_redis_user(self):
 		acl_list = RedisQueue.gen_acl_list()
 		username, password = acl_list[1]["bench"]
@@ -47,6 +50,7 @@ class TestRedisAuth(FrappeTestCase):
 		conn.acl_deluser(username)
 
 	@skip_if_redis_version_lt("6.0")
+	@patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False})
 	def test_rq_namespace(self):
 		"""Make sure that user can access only their respective namespace."""
 		# Current bench ID

frappe/tests/test_safe_exec.py

@@ -1,3 +1,5 @@
+import types
+
 import frappe
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils.safe_exec import get_safe_globals, safe_exec
@@ -59,3 +61,17 @@ class TestSafeExec(FrappeTestCase):
 
 		# enqueue whitelisted method
 		safe_exec("""frappe.enqueue("ping", now=True)""")
+
+	def test_ensure_getattrable_globals(self):
+		def check_safe(objects):
+			for obj in objects:
+				if isinstance(obj, (types.ModuleType, types.CodeType, types.TracebackType, types.FrameType)):
+					self.fail(f"{obj} wont work in safe exec.")
+				elif isinstance(obj, dict):
+					check_safe(obj.values())
+
+		check_safe(get_safe_globals().values())
+
+	def test_unsafe_objects(self):
+		unsafe_global = {"frappe": frappe}
+		self.assertRaises(SyntaxError, safe_exec, """frappe.msgprint("Hello")""", unsafe_global)

frappe/tests/test_utils.py

@@ -493,6 +493,10 @@ class TestDateUtils(FrappeTestCase):
 			frappe.utils.get_last_day_of_week("2020-12-28"), frappe.utils.getdate("2021-01-02")
 		)
 
+	def test_is_last_day_of_the_month(self):
+		self.assertEqual(frappe.utils.is_last_day_of_the_month("2020-12-24"), False)
+		self.assertEqual(frappe.utils.is_last_day_of_the_month("2020-12-31"), True)
+
 	def test_get_time(self):
 		datetime_input = now_datetime()
 		timedelta_input = get_timedelta()

frappe/tests/test_virtual_doctype.py

@@ -87,7 +87,7 @@ class TestVirtualDoctypes(FrappeTestCase):
 		cls.addClassCleanup(frappe.flags.pop, "allow_doctype_export", None)
 
 		vdt = new_doctype(name=TEST_DOCTYPE_NAME, is_virtual=1, custom=0).insert()
-		cls.addClassCleanup(vdt.delete)
+		cls.addClassCleanup(vdt.delete, force=True)
 
 		patch_virtual_doc = patch(
 			"frappe.controllers", new={frappe.local.site: {TEST_DOCTYPE_NAME: VirtualDoctypeTest}}

frappe/tests/test_zbg_job_sanity_test.py

@@ -0,0 +1,27 @@
+""" smoak tests to check that all registered background jobs execute without error.
+
+Note: Filename is intentional to run this test roughly at end. Don't change."""
+
+import time
+
+import frappe
+from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs
+from frappe.tests.utils import FrappeTestCase, timeout
+
+
+class TestScheduledJobSanity(FrappeTestCase):
+	def setUp(self):
+		remove_failed_jobs()
+
+	@timeout(90)
+	def test_bg_jobs_run(self):
+		"""Enqueue all scheduled jobs, wait for finish and verify that none failed."""
+		for scheduled_job_type in frappe.get_all("Scheduled Job Type", pluck="name"):
+			frappe.get_doc("Scheduled Job Type", scheduled_job_type).enqueue(force=True)
+
+		while RQJob.get_list({"filters": [["RQ Job", "status", "in", ("queued", "started")]]}):
+			time.sleep(0.5)
+
+		# Check no failed, if failed print full details
+		failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]})
+		self.assertEqual(len(failed_jobs), 0, "Jobs failed: " + str(failed_jobs))

frappe/tests/ui_test_helpers.py

@@ -451,7 +451,7 @@ def create_test_user():
 
 @frappe.whitelist()
 def setup_tree_doctype():
-	frappe.delete_doc_if_exists("DocType", "Custom Tree")
+	frappe.delete_doc_if_exists("DocType", "Custom Tree", force=True)
 
 	frappe.get_doc(
 		{
@@ -475,7 +475,7 @@ def setup_tree_doctype():
 
 @frappe.whitelist()
 def setup_image_doctype():
-	frappe.delete_doc_if_exists("DocType", "Custom Image")
+	frappe.delete_doc_if_exists("DocType", "Custom Image", force=True)
 
 	frappe.get_doc(
 		{

frappe/translations/de.csv

@@ -4080,7 +4080,7 @@ Scheduler Event,Scheduler-Ereignis,
 Select Event Type,Wählen Sie den Ereignistyp,
 Schedule Script,Zeitplan-Skript,
 Duration,Dauer,
-Donut,Krapfen,
+Donut,Donut,
 Custom Options,Benutzerdefinierte Optionen,
 "Ex: ""colors"": [""#d1d8dd"", ""#ff5858""]","Beispiel: "Farben": ["# d1d8dd", "# ff5858"]",
 Confirmation Email Template,Bestätigungs-E-Mail-Vorlage,
@@ -4818,3 +4818,8 @@ K,Tsd,Number system
 M,Mio,Number system
 B,Mrd,Number system
 T,Bio,Number system
+Type of Chart,Diagrammtyp,
+Preview Chart,Vorschau erzeugen,
+Please select X and Y fields,Bitte Felder für die X- und Y-Achse wählen,
+Notification sent to,Benachrichtigung gesendet an,
+Add to this activity by mailing to {0},"Senden Sie eine E-Mail an {0}, damit sie hier erscheint",

frappe/utils/background_jobs.py

@@ -9,7 +9,6 @@ from uuid import uuid4
 import redis
 from redis.exceptions import BusyLoadingError, ConnectionError
 from rq import Connection, Queue, Worker
-from rq.command import send_stop_job_command
 from rq.logutils import setup_loghandlers
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 

frappe/utils/data.py

@@ -471,6 +471,12 @@ def get_last_day(dt):
 	return get_first_day(dt, 0, 1) + datetime.timedelta(-1)
 
 
+def is_last_day_of_the_month(dt):
+	last_day_of_the_month = get_last_day(dt)
+
+	return getdate(dt) == getdate(last_day_of_the_month)
+
+
 def get_quarter_ending(date):
 	date = getdate(date)
 

frappe/utils/deprecations.py

@@ -0,0 +1,27 @@
+""" Utils for deprecating functionality in Framework.
+
+WARNING: This file is internal, instead of depending just copy the code or use deprecation
+libraries.
+"""
+import functools
+import warnings
+
+
+def deprecated(func):
+	"""Decorator to wrap a function/method as deprecated."""
+
+	@functools.wraps(func)
+	def wrapper(*args, **kwargs):
+		deprecation_warning(
+			f"{func.__name__} is deprecated and will be removed in next major version.",
+			stacklevel=1,
+		)
+		return func(*args, **kwargs)
+
+	return wrapper
+
+
+def deprecation_warning(message, category=DeprecationWarning, stacklevel=1):
+	"""like warnings.warn but with auto incremented sane stacklevel."""
+
+	warnings.warn(message=message, category=category, stacklevel=stacklevel + 2)

frappe/utils/jinja_globals.py

@@ -92,9 +92,7 @@ def web_blocks(blocks):
 def get_dom_id(seed=None):
 	from frappe import generate_hash
 
-	if not seed:
-		seed = "DOM"
-	return "id-" + generate_hash(seed, 12)
+	return "id-" + generate_hash(12)
 
 
 def include_script(path, preload=True):

frappe/utils/oauth.py

@@ -266,7 +266,7 @@ def update_oauth_user(user, data, provider):
 				"email": get_email(data),
 				"gender": gender,
 				"enabled": 1,
-				"new_password": frappe.generate_hash(get_email(data)),
+				"new_password": frappe.generate_hash(),
 				"location": data.get("location"),
 				"user_type": "Website User",
 				"user_image": data.get("picture") or data.get("avatar_url"),

frappe/utils/safe_exec.py

@@ -2,6 +2,9 @@ import copy
 import inspect
 import json
 import mimetypes
+import types
+from contextlib import contextmanager
+from functools import lru_cache
 
 import RestrictedPython.Guards
 from RestrictedPython import compile_restricted, safe_globals
@@ -64,14 +67,20 @@ def safe_exec(script, _globals=None, _locals=None, restrict_commit_rollback=Fals
 		exec_globals.frappe.db.pop("rollback", None)
 		exec_globals.frappe.db.pop("add_index", None)
 
-	# execute script compiled by RestrictedPython
-	frappe.flags.in_safe_exec = True
-	exec(compile_restricted(script), exec_globals, _locals)  # pylint: disable=exec-used
-	frappe.flags.in_safe_exec = False
+	with safe_exec_flags(), patched_qb():
+		# execute script compiled by RestrictedPython
+		exec(compile_restricted(script), exec_globals, _locals)  # pylint: disable=exec-used
 
 	return exec_globals, _locals
 
 
+@contextmanager
+def safe_exec_flags():
+	frappe.flags.in_safe_exec = True
+	yield
+	frappe.flags.in_safe_exec = False
+
+
 def get_safe_globals():
 	datautils = frappe._dict()
 
@@ -258,6 +267,28 @@ def call_with_form_dict(function, kwargs):
 		frappe.local.form_dict = form_dict
 
 
+@contextmanager
+def patched_qb():
+	require_patching = isinstance(frappe.qb.terms, types.ModuleType)
+	try:
+		if require_patching:
+			_terms = frappe.qb.terms
+			frappe.qb.terms = _flatten(frappe.qb.terms)
+		yield
+	finally:
+		if require_patching:
+			frappe.qb.terms = _terms
+
+
+@lru_cache
+def _flatten(module):
+	new_mod = NamespaceDict()
+	for name, obj in inspect.getmembers(module, lambda x: not inspect.ismodule(x)):
+		if not name.startswith("_"):
+			new_mod[name] = obj
+	return new_mod
+
+
 def get_python_builtins():
 	return {
 		"abs": abs,
@@ -350,6 +381,10 @@ def _getattr(object, name, default=None):
 
 	if isinstance(name, str) and (name in UNSAFE_ATTRIBUTES):
 		raise SyntaxError(f"{name} is an unsafe attribute")
+
+	if isinstance(object, (types.ModuleType, types.CodeType, types.TracebackType, types.FrameType)):
+		raise SyntaxError(f"Reading {object} attributes is not allowed")
+
 	return RestrictedPython.Guards.safer_getattr(object, name, default=default)
 
 

frappe/utils/verified_command.py

@@ -16,15 +16,15 @@ def get_signed_params(params):
 	if not isinstance(params, str):
 		params = urlencode(params)
 
-	signature = hmac.new(params.encode(), digestmod=hashlib.md5)
+	signature = hmac.new(params.encode(), digestmod=hashlib.sha512)
 	signature.update(get_secret().encode())
 	return params + "&_signature=" + signature.hexdigest()
 
 
 def get_secret():
-	return frappe.local.conf.get("secret") or str(
-		frappe.db.get_value("User", "Administrator", "creation")
-	)
+	from frappe.utils.password import get_encryption_key
+
+	return frappe.local.conf.get("secret") or get_encryption_key()
 
 
 def verify_request():
@@ -37,10 +37,10 @@ def verify_request():
 	if signature_string in query_string:
 		params, signature = query_string.split(signature_string)
 
-		given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.md5)
+		given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.sha512)
 
 		given_signature.update(get_secret().encode())
-		valid_signature = signature == given_signature.hexdigest()
+		valid_signature = hmac.compare_digest(signature, given_signature.hexdigest())
 		valid_method = frappe.request.method == "GET"
 		valid_request_data = not (frappe.request.form or frappe.request.data)
 

frappe/utils/weasyprint.py

@@ -9,6 +9,7 @@ import frappe
 @frappe.whitelist()
 def download_pdf(doctype, name, print_format, letterhead=None):
 	doc = frappe.get_doc(doctype, name)
+	doc.check_permission("print")
 	generator = PrintFormatGenerator(print_format, doc, letterhead)
 	pdf = generator.render_pdf()
 
@@ -21,6 +22,7 @@ def download_pdf(doctype, name, print_format, letterhead=None):
 
 def get_html(doctype, name, print_format, letterhead=None):
 	doc = frappe.get_doc(doctype, name)
+	doc.check_permission("print")
 	generator = PrintFormatGenerator(print_format, doc, letterhead)
 	return generator.get_html_preview()
 

frappe/website/doctype/website_theme/website_theme.py

@@ -75,7 +75,7 @@ class WebsiteTheme(Document):
 			self.delete_old_theme_files(folder_path)
 
 		# add a random suffix
-		suffix = frappe.generate_hash("Website Theme", 8) if self.custom else "style"
+		suffix = frappe.generate_hash(length=8) if self.custom else "style"
 		file_name = frappe.scrub(self.name) + "_" + suffix + ".css"
 		output_path = join_path(folder_path, file_name)
 

frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html

@@ -7,7 +7,7 @@
 	<div class="collapsible-items">
 		{%- for item in items -%}
 		<div class="collapsible-item">
-			{%- set collapse_id = 'id-' + frappe.utils.generate_hash('Collapse', 12) -%}
+			{%- set collapse_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
 			<a class="collapsible-title" data-toggle="collapse" href="#{{ collapse_id }}" role="button"
 				aria-expanded="false" aria-controls="{{ collapse_id }}">
 				<div class="collapsible-item-title">{{ _(item.title) }}</div>

frappe/website/web_template/section_with_tabs/section_with_tabs.html

@@ -11,8 +11,8 @@
 
 	{%- for index in ['1', '2', '3', '4', '5', '6'] -%}
 
-	{%- set buttonid = 'id-' + frappe.utils.generate_hash('TabButton', 12) -%}
-	{%- set panelid = 'id-' + frappe.utils.generate_hash('TabPanel', 12) -%}
+	{%- set buttonid = 'id-' + frappe.utils.generate_hash(length=12) -%}
+	{%- set panelid = 'id-' + frappe.utils.generate_hash(length=12) -%}
 
 	{%- set tab = {
 		'title': values['tab_' + index + '_title'],

frappe/website/web_template/slideshow/slideshow.html

@@ -1,6 +1,6 @@
 {%- set slideshow = frappe.get_doc('Website Slideshow', website_slideshow) -%}
 {%- set slides = slideshow.slideshow_items -%}
-{%- set slideshow_id = 'id-' + frappe.utils.generate_hash('Slideshow', 12) -%}
+{%- set slideshow_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
 
 {{ slideshow.header or '' }}
 

pyproject.toml

@@ -57,7 +57,7 @@ dependencies = [
     "hiredis~=2.0.0",
     "requests-oauthlib~=1.3.0",
     "requests~=2.27.1",
-    "rq~=1.10.1",
+    "rq~=1.11.1",
     "rsa>=4.1",
     "semantic-version~=2.10.0",
     "sqlparse~=0.4.1",
@@ -93,7 +93,7 @@ ensure_newline_before_comments = true
 indent = "\t"
 
 [tool.bench.dev-dependencies]
-coverage = "~=6.4.1"
+coverage = "~=6.5.0"
 Faker = "~=13.12.1"
 pyngrok = "~=5.0.5"
 unittest-xml-reporting = "~=3.0.4"

socketio.js

@@ -31,11 +31,6 @@ io.on("connection", function (socket) {
 
 	socket.user = cookie.parse(socket.request.headers.cookie).user_id;
 
-	socket.on("task_subscribe", function (task_id) {
-		var room = get_task_room(socket, task_id);
-		socket.join(room);
-	});
-
 	let retries = 0;
 	let join_user_room = () => {
 		request
@@ -61,13 +56,13 @@ io.on("connection", function (socket) {
 
 	join_user_room();
 
-	socket.on("task_unsubscribe", function (task_id) {
+	socket.on("task_subscribe", function (task_id) {
 		var room = get_task_room(socket, task_id);
-		socket.leave(room);
+		socket.join(room);
 	});
 
 	socket.on("task_unsubscribe", function (task_id) {
-		var room = "task:" + task_id;
+		var room = get_task_room(socket, task_id);
 		socket.leave(room);
 	});
 

yarn.lock

@@ -3170,9 +3170,9 @@ socket.io-client@^4.5.1:
     socket.io-parser "~4.2.0"
 
 socket.io-parser@~4.0.4:
-  version "4.0.4"
-  resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.0.4.tgz#9ea21b0d61508d18196ef04a2c6b9ab630f4c2b0"
-  integrity sha512-t+b0SS+IxG7Rxzda2EVvyBZbvFPBCjJoyHuE0P//7OAsN23GItzDRdWa6ALxZI/8R5ygK7jAR6t028/z+7295g==
+  version "4.0.5"
+  resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.0.5.tgz#cb404382c32324cc962f27f3a44058cf6e0552df"
+  integrity sha512-sNjbT9dX63nqUFIOv95tTVm6elyIU4RvB1m8dOeZt+IgWwcWklFDOdmGcfo3zSiRsnR/3pJkjY5lfoGqEe4Eig==
   dependencies:
     "@types/component-emitter" "^1.2.10"
     component-emitter "~1.3.0"