Merge branch 'frappe:develop' into multiple-role-profile

2024-02-05 12:22:49 +05:30 · 2024-02-05 12:22:49 +05:30 · 1909fcdc22
commit 1909fcdc22
parent cd24f32239 985c95e900
110 changed files with 2577 additions and 1744 deletions
--- a/.github/helper/roulette.py
+++ b/.github/helper/roulette.py
@ -73,8 +73,9 @@ def has_label(pr_number, label, repo="frappe/frappe"):
 	)


-def is_py(file):
-	return file.endswith("py")
+def is_server_side_code(file):
+	"""File exclusively affects server side code"""
+	return file.endswith("py") or file.endswith(".po")


 def is_ci(file):
@ -112,7 +113,7 @@ if __name__ == "__main__":
 	ci_files_changed = any(f for f in files_list if is_ci(f))
 	only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list)
 	only_frontend_code_changed = len(list(filter(is_frontend_code, files_list))) == len(files_list)
-	updated_py_file_count = len(list(filter(is_py, files_list)))
+	updated_py_file_count = len(list(filter(is_server_side_code, files_list)))
 	only_py_changed = updated_py_file_count == len(files_list)

 	if has_skip_ci_label(pr_number, repo):
--- a/.github/workflows/server-tests.yml
+++ b/.github/workflows/server-tests.yml
@ -43,6 +43,8 @@ jobs:
    needs: checkrun
    if: ${{ needs.checkrun.outputs.build == 'strawberry' }}
    timeout-minutes: 60
+    env:
+      NODE_ENV: "production"

    strategy:
      fail-fast: false
--- a/.github/workflows/ui-tests.yml
+++ b/.github/workflows/ui-tests.yml
@ -42,6 +42,8 @@ jobs:
    needs: checkrun
    if: ${{ needs.checkrun.outputs.build == 'strawberry' && github.repository_owner == 'frappe' }}
    timeout-minutes: 60
+    env:
+      NODE_ENV: "production"

    strategy:
      fail-fast: false
--- a/babel_extractors.csv
+++ b/babel_extractors.csv
@ -6,4 +6,4 @@ hooks.py,frappe.gettext.extractors.navbar.extract
 **/report/*/*.json,frappe.gettext.extractors.report.extract
 **.py,frappe.gettext.extractors.python.extract
 **.js,frappe.gettext.extractors.javascript.extract
-**.html,frappe.gettext.extractors.jinja2.extract
+**.html,frappe.gettext.extractors.html_template.extract
--- a/frappe/init.py
+++ b/frappe/init.py
@ -130,9 +130,45 @@ def _lt(msg: str, lang: str | None = None, context: str | None = None):

 	Note: Result is not guaranteed to equivalent to pure strings for all operations.
 	"""
-	from frappe.translate import LazyTranslate
+	return _LazyTranslate(msg, lang, context)

-	return LazyTranslate(msg, lang, context)
+
+@functools.total_ordering
+class _LazyTranslate:
+	__slots__ = ("msg", "lang", "context")
+
+	def __init__(self, msg: str, lang: str | None = None, context: str | None = None) -> None:
+		self.msg = msg
+		self.lang = lang
+		self.context = context
+
+	@property
+	def value(self) -> str:
+		return _(str(self.msg), self.lang, self.context)
+
+	def __str__(self):
+		return self.value
+
+	def __add__(self, other):
+		if isinstance(other, (str, _LazyTranslate)):
+			return self.value + str(other)
+		raise NotImplementedError
+
+	def __radd__(self, other):
+		if isinstance(other, (str, _LazyTranslate)):
+			return str(other) + self.value
+		return NotImplementedError
+
+	def __repr__(self) -> str:
+		return f"'{self.value}'"
+
+	# NOTE: it's required to override these methods and raise error as default behaviour will
+	# return `False` in all cases.
+	def __eq__(self, other):
+		raise NotImplementedError
+
+	def __lt__(self, other):
+		raise NotImplementedError


 def as_unicode(text, encoding: str = "utf-8") -> str:
@ -269,10 +305,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False)

 	local.initialised = True

-	# Set the user as database name if not set in config
-	if local.conf and local.conf.db_name is not None and local.conf.db_user is None:
-		local.conf.db_user = local.conf.db_name
-

 def connect(
 	site: str | None = None, db_name: str | None = None, set_admin_as_user: bool = True
@ -280,7 +312,7 @@ def connect(
 	"""Connect to site database instance.

 	:param site: (Deprecated) If site is given, calls `frappe.init`.
-	:param db_name: Optional. Will use from `site_config.json`.
+	:param db_name: (Deprecated) Optional. Will use from `site_config.json`.
 	:param set_admin_as_user: Set Administrator as current user.
 	"""
 	from frappe.database import get_db
@ -293,13 +325,24 @@ def connect(
 			"Instead, explicitly invoke frappe.init(site) prior to calling frappe.connect(), if initializing the site is necessary."
 		)
 		init(site)
+	if db_name:
+		from frappe.utils.deprecations import deprecation_warning
+
+		deprecation_warning(
+			"Calling frappe.connect with the db_name argument is deprecated and will be removed in next major version. "
+			"Instead, explicitly invoke frappe.init(site) with the right config prior to calling frappe.connect(), if necessary."
+		)
+
+	assert db_name or local.conf.db_user, "site must be fully initialized, db_user missing"
+	assert db_name or local.conf.db_name, "site must be fully initialized, db_name missing"
+	assert local.conf.db_password, "site must be fully initialized, db_password missing"

 	local.db = get_db(
 		host=local.conf.db_host,
 		port=local.conf.db_port,
-		user=local.conf.db_user or db_name or local.conf.db_name,
+		user=local.conf.db_user or db_name,
 		password=local.conf.db_password,
-		cur_db_name=db_name or local.conf.db_name,
+		cur_db_name=local.conf.db_name or db_name,
 	)
 	if set_admin_as_user:
 		set_user("Administrator")
@ -387,6 +430,11 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 		os.environ.get("FRAPPE_DB_PORT") or config.get("db_port") or db_default_ports(config["db_type"])
 	)

+	# Set the user as database name if not set in config
+	config["db_user"] = (
+		os.environ.get("FRAPPE_DB_USER") or config.get("db_user") or config.get("db_name")
+	)
+
 	return config


--- a/frappe/auth.py
+++ b/frappe/auth.py
@ -274,9 +274,7 @@ class LoginManager:
 		if self.user in frappe.STANDARD_USERS:
 			return False

-		reset_pwd_after_days = cint(
-			frappe.db.get_single_value("System Settings", "force_user_to_reset_password")
-		)
+		reset_pwd_after_days = cint(frappe.get_system_settings("force_user_to_reset_password"))

 		if reset_pwd_after_days:
 			last_password_reset_date = (
--- a/frappe/automation/doctype/auto_repeat/auto_repeat.py
+++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py
@ -550,7 +550,7 @@ def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters
 	docs += [r.name for r in res]
 	docs = set(list(docs))

-	return [[d] for d in docs]
+	return [[d] for d in docs if txt in d]


@frappe.whitelist()
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@ -772,12 +772,8 @@ def run_tests(
 			click.secho(f"bench --site {site} set-config allow_tests true", fg="green")
 			return

-		frappe.init(site=site)
-
-		frappe.flags.skip_before_tests = skip_before_tests
-		frappe.flags.skip_test_records = skip_test_records
-
 		ret = frappe.test_runner.main(
+			site,
 			app,
 			module,
 			doctype,
@ -790,6 +786,8 @@ def run_tests(
 			doctype_list_path=doctype_list_path,
 			failfast=failfast,
 			case=case,
+			skip_test_records=skip_test_records,
+			skip_before_tests=skip_before_tests,
 		)

 		if len(ret.failures) == 0 and len(ret.errors) == 0:
--- a/frappe/contacts/doctype/address_template/address_template.py
+++ b/frappe/contacts/doctype/address_template/address_template.py
@ -28,7 +28,7 @@ class AddressTemplate(Document):

 		if not self.is_default and not self._get_previous_default():
 			self.is_default = 1
-			if frappe.db.get_single_value("System Settings", "setup_complete"):
+			if frappe.get_system_settings("setup_complete"):
 				frappe.msgprint(_("Setting this Address Template as default as there is no other default"))

 	def on_update(self):
--- a/frappe/core/doctype/communication/mixins.py
+++ b/frappe/core/doctype/communication/mixins.py
@ -146,7 +146,7 @@ class CommunicationEmailMixin:
 		return get_formatted_email(self.mail_sender_fullname(), mail=self.mail_sender())

 	def get_content(self, print_format=None):
-		if print_format and frappe.db.get_single_value("System Settings", "attach_view_link"):
+		if print_format and frappe.get_system_settings("attach_view_link"):
 			return self.content + self.get_attach_link(print_format)
 		return self.content

--- a/frappe/core/doctype/data_export/data_export.js
+++ b/frappe/core/doctype/data_export/data_export.js
@ -153,7 +153,7 @@ const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {

 	const options = fields.map((df) => {
 		return {
-			label: df.label,
+			label: __(df.label),
 			value: df.fieldname,
 			danger: df.reqd,
 			checked: 1,
@ -163,7 +163,7 @@ const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {
 	const multicheck_control = frappe.ui.form.make_control({
 		parent: parent_wrapper,
 		df: {
-			label: doctype,
+			label: __(doctype),
 			fieldname: doctype + "_fields",
 			fieldtype: "MultiCheck",
 			options: options,
--- a/frappe/core/doctype/data_import/data_import.js
+++ b/frappe/core/doctype/data_import/data_import.js
@ -135,34 +135,29 @@ frappe.ui.form.on("Data Import", {
 				let failed_records = cint(r.message.failed);
 				let total_records = cint(r.message.total_records);

-				if (!total_records) return;
-				let action, message;
-				if (frm.doc.import_type === "Insert New Records") {
-					action = "imported";
-				} else {
-					action = "updated";
+				if (!total_records) {
+					return;
 				}

-				if (failed_records === 0) {
-					let message_args = [action, successful_records];
-					if (successful_records === 1) {
-						message = __("Successfully {0} 1 record.", message_args);
-					} else {
-						message = __("Successfully {0} {1} records.", message_args);
-					}
+				let message;
+				if (frm.doc.import_type === "Insert New Records") {
+					message = __("Successfully imported {0} out of {1} records.", [
+						successful_records,
+						total_records,
+					]);
 				} else {
-					let message_args = [action, successful_records, total_records];
-					if (successful_records === 1) {
-						message = __(
-							"Successfully {0} {1} record out of {2}. Click on Export Errored Rows, fix the errors and import again.",
-							message_args
+					message = __("Successfully updated {0} out of {1} records.", [
+						successful_records,
+						total_records,
+					]);
+				}
+
+				if (failed_records > 0) {
+					message +=
+						"<br/>" +
+						__(
+							"Please click on 'Export Errored Rows', fix the errors and import again."
 						);
-					} else {
-						message = __(
-							"Successfully {0} {1} records out of {2}. Click on Export Errored Rows, fix the errors and import again.",
-							message_args
-						);
-					}
 				}

 				// If the job timed out, display an extra hint
@ -506,13 +501,7 @@ frappe.ui.form.on("Data Import", {
 	},

 	show_import_log(frm) {
-		if (!frm.doc.show_failed_logs) {
-			frm.toggle_display("import_log_preview", false);
-			return;
-		}
-
 		frm.toggle_display("import_log_section", false);
-		frm.toggle_display("import_log_preview", true);

 		if (frm.import_in_progress) {
 			return;
--- a/frappe/core/doctype/data_import/data_import.json
+++ b/frappe/core/doctype/data_import/data_import.json
@ -139,7 +139,7 @@
   "default": "0",
   "fieldname": "show_failed_logs",
   "fieldtype": "Check",
-   "label": "Show Failed Logs"
+   "label": "Show Only Failed Logs"
  },
  {
   "depends_on": "eval:!doc.__islocal && !doc.import_file",
@ -171,7 +171,7 @@
 ],
 "hide_toolbar": 1,
 "links": [],
- "modified": "2023-12-15 12:45:49.452834",
+ "modified": "2024-01-30 17:08:05.566686",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Data Import",
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@ -38,7 +38,6 @@ class DataImport(Document):
 		submit_after_import: DF.Check
 		template_options: DF.Code | None
 		template_warnings: DF.Code | None
-
 	# end: auto-generated types

 	def validate(self):
@ -93,7 +92,8 @@ class DataImport(Document):
 	def start_import(self):
 		from frappe.utils.scheduler import is_scheduler_inactive

-		if is_scheduler_inactive() and not frappe.flags.in_test:
+		run_now = frappe.flags.in_test or frappe.conf.developer_mode
+		if is_scheduler_inactive() and not run_now:
 			frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive"))

 		job_id = f"data_import::{self.name}"
@ -106,7 +106,7 @@ class DataImport(Document):
 				event="data_import",
 				job_id=job_id,
 				data_import=self.name,
-				now=frappe.conf.developer_mode or frappe.flags.in_test,
+				now=run_now,
 			)
 			return True

--- a/frappe/core/doctype/docfield/docfield.json
+++ b/frappe/core/doctype/docfield/docfield.json
@ -157,6 +157,7 @@
  },
  {
   "default": "0",
+   "depends_on": "eval:!doc.is_virtual",
   "fieldname": "in_list_view",
   "fieldtype": "Check",
   "label": "In List View",
@ -580,7 +581,7 @@
 "index_web_pages_for_search": 1,
 "istable": 1,
 "links": [],
- "modified": "2023-11-16 11:26:56.364594",
+ "modified": "2024-02-01 15:55:44.007917",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "DocField",
--- a/frappe/core/doctype/docfield/docfield.py
+++ b/frappe/core/doctype/docfield/docfield.py
@ -117,6 +117,7 @@ class DocField(Document):
 		unique: DF.Check
 		width: DF.Data | None
 	# end: auto-generated types
+
 	def get_link_doctype(self):
 		"""Return the Link doctype for the `docfield` (if applicable).

--- a/frappe/core/doctype/docshare/test_docshare.py
+++ b/frappe/core/doctype/docshare/test_docshare.py
@ -56,6 +56,24 @@ class TestDocShare(FrappeTestCase):
 		with self.assertRowsRead(1):
 			self.assertTrue(self.event.has_permission())

+	def test_list_permission(self):
+		frappe.set_user(self.user)
+		with self.assertRaises(frappe.PermissionError):
+			frappe.get_list("Web Page")
+
+		frappe.set_user("Administrator")
+		doc = frappe.new_doc("Web Page")
+		doc.update({"title": "test document for docshare permissions"})
+		doc.insert()
+		frappe.share.add("Web Page", doc.name, self.user)
+
+		frappe.set_user(self.user)
+		self.assertEqual(len(frappe.get_list("Web Page")), 1)
+
+		doc.delete(ignore_permissions=True)
+		with self.assertRaises(frappe.PermissionError):
+			frappe.get_list("Web Page")
+
 	def test_share_permission(self):
 		frappe.share.add("Event", self.event.name, self.user, write=1, share=1)

--- a/frappe/core/doctype/file/file.py
+++ b/frappe/core/doctype/file/file.py
@ -756,6 +756,13 @@ class File(Document):
 		self.save_file(content=optimized_content, overwrite=True)
 		self.save()

+	@property
+	def unique_url(self) -> str:
+		"""Unique URL contains file ID in URL to speed up permisison checks."""
+		from urllib.parse import urlencode
+
+		return self.file_url + "?" + urlencode({"fid": self.name})
+
 	@staticmethod
 	def zip_files(files):
 		zip_file = io.BytesIO()
--- a/frappe/core/doctype/recorder/recorder.json
+++ b/frappe/core/doctype/recorder/recorder.json
@ -20,7 +20,9 @@
  "section_break_sgro",
  "form_dict",
  "section_break_9jhm",
-  "sql_queries"
+  "sql_queries",
+  "section_break_optn",
+  "profile"
 ],
 "fields": [
  {
@ -107,6 +109,16 @@
   "fieldtype": "Data",
   "hidden": 1,
   "label": "Event Type"
+  },
+  {
+   "fieldname": "section_break_optn",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "profile",
+   "fieldtype": "Code",
+   "label": "cProfile Output",
+   "read_only": 1
  }
 ],
 "hide_toolbar": 1,
@ -114,7 +126,7 @@
 "index_web_pages_for_search": 1,
 "is_virtual": 1,
 "links": [],
- "modified": "2024-01-03 16:45:47.110048",
+ "modified": "2024-02-01 22:13:26.505174",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Recorder",
--- a/frappe/core/doctype/recorder/recorder.py
+++ b/frappe/core/doctype/recorder/recorder.py
@ -24,6 +24,7 @@ class Recorder(Document):
 		method: DF.Literal["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"]
 		number_of_queries: DF.Int
 		path: DF.Data | None
+		profile: DF.Code | None
 		request_headers: DF.Code | None
 		sql_queries: DF.Table[RecorderQuery]
 		time: DF.Datetime | None
--- a/frappe/core/doctype/recorder/recorder_list.js
+++ b/frappe/core/doctype/recorder/recorder_list.js
@ -10,12 +10,7 @@ frappe.listview_settings["Recorder"] = {
 		}

 		listview.page.add_button(__("Clear"), () => {
-			frappe.call({
-				method: "frappe.recorder.delete",
-				callback: function () {
-					listview.refresh();
-				},
-			});
+			frappe.xcall("frappe.recorder.delete").then(listview.refresh);
 		});

 		listview.page.add_menu_item(__("Import"), () => {
@ -88,18 +83,125 @@ frappe.listview_settings["Recorder"] = {
 	},

 	setup_recorder_controls(listview) {
+		let me = this;
 		listview.page.set_primary_action(listview.enabled ? __("Stop") : __("Start"), () => {
-			frappe.call({
-				method: listview.enabled ? "frappe.recorder.stop" : "frappe.recorder.start",
-				callback: function () {
-					listview.refresh();
-				},
-			});
-			listview.enabled = !listview.enabled;
-			this.refresh_controls(listview);
+			if (listview.enabled) {
+				me.stop_recorder(listview);
+			} else {
+				me.start_recorder(listview);
+			}
 		});
 	},

+	stop_recorder(listview) {
+		let me = this;
+		frappe.xcall("frappe.recorder.stop", {}).then(() => {
+			listview.refresh();
+			listview.enabled = false;
+			me.refresh_controls(listview);
+		});
+	},
+
+	start_recorder(listview) {
+		let me = this;
+		frappe.prompt(
+			[
+				{
+					fieldtype: "Section Break",
+					fieldname: "req_job_section",
+				},
+				{
+					fieldtype: "Column Break",
+					fieldname: "web_request_columns",
+					label: "Web Requests",
+				},
+				{
+					fieldname: "record_requests",
+					fieldtype: "Check",
+					label: "Record Web Requests",
+					default: 1,
+				},
+				{
+					fieldname: "request_filter",
+					fieldtype: "Data",
+					label: "Request path filter",
+					default: "/",
+					depends_on: "record_requests",
+					description: `This will be used for filtering paths which will be recorded.
+						You can use this to avoid slowing down other traffic.
+						e.g. <code>/api/method/erpnext</code>. Leave it empty to record every request.`,
+				},
+				{
+					fieldtype: "Column Break",
+					fieldname: "background_col",
+					label: "Background Jobs",
+				},
+
+				{
+					fieldname: "record_jobs",
+					fieldtype: "Check",
+					label: "Record Background Jobs",
+					default: 1,
+				},
+				{
+					fieldname: "jobs_filter",
+					fieldtype: "Data",
+					label: "Background Jobs filter",
+					default: "",
+					depends_on: "record_jobs",
+					description: `This will be used for filtering jobs which will be recorded.
+						You can use this to avoid slowing down other jobs. e.g. <code>email_queue.pull</code>.
+						Leave it empty to record every job.`,
+				},
+				{
+					fieldtype: "Section Break",
+					fieldname: "sql_section",
+					label: "SQL",
+				},
+				{
+					fieldname: "record_sql",
+					fieldtype: "Check",
+					label: "Record SQL queries",
+					default: 1,
+				},
+				{
+					fieldname: "explain",
+					fieldtype: "Check",
+					label: "Generate EXPLAIN for SQL queries",
+					default: 1,
+				},
+				{
+					fieldname: "capture_stack",
+					fieldtype: "Check",
+					label: "Capture callstack of SQL queries",
+					default: 1,
+				},
+				{
+					fieldtype: "Section Break",
+					fieldname: "python_section",
+					label: "Python",
+				},
+				{
+					fieldname: "profile",
+					fieldtype: "Check",
+					label: "Run cProfile",
+					default: 0,
+					description:
+						"Warning: cProfile adds a lot of overhead. For best results, disable stack capturing when using cProfile.",
+				},
+			],
+			(values) => {
+				frappe.xcall("frappe.recorder.start", values).then(() => {
+					listview.refresh();
+					listview.enabled = true;
+					me.refresh_controls(listview);
+				});
+			},
+			__("Configure Recorder"),
+			__("Start Recordig")
+		);
+	},
+
 	update_indicators(listview) {
 		if (listview.enabled) {
 			listview.page.set_indicator(__("Active"), "green");
--- a/frappe/core/doctype/role_profile/role_profile.py
+++ b/frappe/core/doctype/role_profile/role_profile.py
@ -25,7 +25,11 @@ class RoleProfile(Document):
 		self.name = self.role_profile

 	def on_update(self):
-		self.queue_action("update_all_users", now=frappe.flags.in_test, enqueue_after_commit=True)
+		self.queue_action(
+			"update_all_users",
+			now=frappe.flags.in_test or frappe.flags.in_install,
+			enqueue_after_commit=True,
+		)

 	def update_all_users(self):
 		"""Changes in role_profile reflected across all its user"""
--- a/frappe/core/doctype/server_script/test_server_script.py
+++ b/frappe/core/doctype/server_script/test_server_script.py
@ -238,7 +238,6 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
 		script.execute_method()

 	def test_server_script_rate_limiting(self):
-		# why not
 		script1 = frappe.get_doc(
 			doctype="Server Script",
 			name="rate_limited_server_script",
--- a/frappe/core/doctype/system_settings/system_settings.py
+++ b/frappe/core/doctype/system_settings/system_settings.py
@ -132,6 +132,9 @@ class SystemSettings(Document):
 		self.validate_backup_limit()
 		self.validate_file_extensions()

+		if not self.link_field_results_limit:
+			self.link_field_results_limit = 10
+
 		if self.link_field_results_limit > 50:
 			self.link_field_results_limit = 50
 			label = _(self.meta.get_label("link_field_results_limit"))
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@ -731,12 +731,8 @@ class User(Document):
 		3. If allow_login_using_user_name is set, you can use username while finding the user.
 		"""

-		login_with_mobile = cint(
-			frappe.db.get_single_value("System Settings", "allow_login_using_mobile_number")
-		)
-		login_with_username = cint(
-			frappe.db.get_single_value("System Settings", "allow_login_using_user_name")
-		)
+		login_with_mobile = cint(frappe.get_system_settings("allow_login_using_mobile_number"))
+		login_with_username = cint(frappe.get_system_settings("allow_login_using_user_name"))

 		or_filters = [{"name": user_name}]
 		if login_with_mobile:
@ -846,8 +842,8 @@ def update_password(
 	else:
 		user = res["user"]

-	logout_all_sessions = cint(logout_all_sessions) or frappe.db.get_single_value(
-		"System Settings", "logout_on_password_reset"
+	logout_all_sessions = cint(logout_all_sessions) or frappe.get_system_settings(
+		"logout_on_password_reset"
 	)
 	_update_password(user, new_password, logout_all_sessions=cint(logout_all_sessions))

@ -939,7 +935,7 @@ def _get_user_for_update_password(key, old_password):
 		result.user, last_reset_password_key_generated_on = user or (None, None)
 		if result.user:
 			reset_password_link_expiry = cint(
-				frappe.db.get_single_value("System Settings", "reset_password_link_expiry_duration")
+				frappe.get_system_settings("reset_password_link_expiry_duration")
 			)
 			if (
 				reset_password_link_expiry
@ -1024,7 +1020,7 @@ def sign_up(email: str, full_name: str, redirect_to: str) -> tuple[int, str]:


@frappe.whitelist(allow_guest=True)
-@rate_limit(limit=get_password_reset_limit, seconds=24 * 60 * 60)
+@rate_limit(limit=get_password_reset_limit, seconds=60 * 60)
 def reset_password(user: str) -> str:
 	if user == "Administrator":
 		return "not allowed"
@ -1265,34 +1261,37 @@ def create_contact(user, ignore_links=False, ignore_mandatory=False):
 		except frappe.DuplicateEntryError:
 			pass
 	else:
-		contact = frappe.get_doc("Contact", contact_name)
-		contact.first_name = user.first_name
-		contact.last_name = user.last_name
-		contact.gender = user.gender
+		try:
+			contact = frappe.get_doc("Contact", contact_name)
+			contact.first_name = user.first_name
+			contact.last_name = user.last_name
+			contact.gender = user.gender

-		# Add mobile number if phone does not exists in contact
-		if user.phone and not any(new_contact.phone == user.phone for new_contact in contact.phone_nos):
-			# Set primary phone if there is no primary phone number
-			contact.add_phone(
-				user.phone,
-				is_primary_phone=not any(
-					new_contact.is_primary_phone == 1 for new_contact in contact.phone_nos
-				),
-			)
+			# Add mobile number if phone does not exists in contact
+			if user.phone and not any(new_contact.phone == user.phone for new_contact in contact.phone_nos):
+				# Set primary phone if there is no primary phone number
+				contact.add_phone(
+					user.phone,
+					is_primary_phone=not any(
+						new_contact.is_primary_phone == 1 for new_contact in contact.phone_nos
+					),
+				)

-		# Add mobile number if mobile does not exists in contact
-		if user.mobile_no and not any(
-			new_contact.phone == user.mobile_no for new_contact in contact.phone_nos
-		):
-			# Set primary mobile if there is no primary mobile number
-			contact.add_phone(
-				user.mobile_no,
-				is_primary_mobile_no=not any(
-					new_contact.is_primary_mobile_no == 1 for new_contact in contact.phone_nos
-				),
-			)
+			# Add mobile number if mobile does not exists in contact
+			if user.mobile_no and not any(
+				new_contact.phone == user.mobile_no for new_contact in contact.phone_nos
+			):
+				# Set primary mobile if there is no primary mobile number
+				contact.add_phone(
+					user.mobile_no,
+					is_primary_mobile_no=not any(
+						new_contact.is_primary_mobile_no == 1 for new_contact in contact.phone_nos
+					),
+				)

-		contact.save(ignore_permissions=True)
+			contact.save(ignore_permissions=True)
+		except frappe.TimestampMismatchError:
+			raise frappe.RetryBackgroundJobError


 def get_restricted_ip_list(user):
--- a/frappe/core/page/permission_manager/permission_manager.js
+++ b/frappe/core/page/permission_manager/permission_manager.js
@ -253,6 +253,10 @@ frappe.PermissionEngine = class PermissionEngine {
 				if (!d.is_submittable && ["submit", "cancel", "amend"].includes(r)) return;
 				if (d.in_create && ["create", "delete"].includes(r)) return;
 				this.add_check(perm_container, d, r);
+
+				if (d.if_owner && r == "report") {
+					perm_container.find("div[data-fieldname='report']").toggle(false);
+				}
 			});

 			// buttons
@ -414,6 +418,13 @@ frappe.PermissionEngine = class PermissionEngine {
 						chk.prop("checked", !chk.prop("checked"));
 					} else {
 						me.get_perm(args.role)[args.ptype] = args.value;
+
+						if (args.ptype == "if_owner") {
+							let report_checkbox = chk
+								.closest("div.row")
+								.find("div[data-fieldname='report']");
+							report_checkbox.toggle(!args.value);
+						}
 					}
 				},
 			});
--- a/frappe/core/page/permission_manager/permission_manager.py
+++ b/frappe/core/page/permission_manager/permission_manager.py
@ -129,8 +129,15 @@ def update(
 		frappe.clear_cache(doctype=doctype)

 	frappe.only_for("System Manager")
+
+	if ptype == "report" and value == "1" and if_owner == "1":
+		frappe.throw(_("Cannot set 'Report' permission if 'Only If Creator' permission is set"))
+
 	out = update_permission_property(doctype, role, permlevel, ptype, value, if_owner=if_owner)

+	if ptype == "if_owner" and value == "1":
+		update_permission_property(doctype, role, permlevel, "report", "0", if_owner=value)
+
 	frappe.db.after_commit.add(clear_cache)

 	return "refresh" if out else None
--- a/frappe/custom/doctype/custom_field/custom_field.js
+++ b/frappe/custom/doctype/custom_field/custom_field.js
@ -112,28 +112,30 @@ frappe.ui.form.on("Custom Field", {
 		}
 	},
 	add_rename_field(frm) {
-		frm.add_custom_button(__("Rename Fieldname"), () => {
-			frappe.prompt(
-				{
-					fieldtype: "Data",
-					label: __("Fieldname"),
-					fieldname: "fieldname",
-					reqd: 1,
-					default: frm.doc.fieldname,
-				},
-				function (data) {
-					frappe.call({
-						method: "frappe.custom.doctype.custom_field.custom_field.rename_fieldname",
-						args: {
-							custom_field: frm.doc.name,
-							fieldname: data.fieldname,
-						},
-					});
-				},
-				__("Rename Fieldname"),
-				__("Rename")
-			);
-		});
+		if (!frm.is_new()) {
+			frm.add_custom_button(__("Rename Fieldname"), () => {
+				frappe.prompt(
+					{
+						fieldtype: "Data",
+						label: __("Fieldname"),
+						fieldname: "fieldname",
+						reqd: 1,
+						default: frm.doc.fieldname,
+					},
+					function (data) {
+						frappe.call({
+							method: "frappe.custom.doctype.custom_field.custom_field.rename_fieldname",
+							args: {
+								custom_field: frm.doc.name,
+								fieldname: data.fieldname,
+							},
+						});
+					},
+					__("Rename Fieldname"),
+					__("Rename")
+				);
+			});
+		}
 	},
 });

--- a/frappe/custom/doctype/customize_form_field/customize_form_field.json
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.json
@ -130,6 +130,7 @@
  },
  {
   "default": "0",
+   "depends_on": "eval:!doc.is_virtual",
   "fieldname": "in_list_view",
   "fieldtype": "Check",
   "label": "In List View"
@ -483,7 +484,7 @@
 "index_web_pages_for_search": 1,
 "istable": 1,
 "links": [],
- "modified": "2023-12-08 15:52:37.525003",
+ "modified": "2024-02-01 15:56:39.171633",
 "modified_by": "Administrator",
 "module": "Custom",
 "name": "Customize Form Field",
--- a/frappe/custom/doctype/customize_form_field/customize_form_field.py
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.py
@ -110,4 +110,5 @@ class CustomizeFormField(Document):
 		unique: DF.Check
 		width: DF.Data | None
 	# end: auto-generated types
+
 	pass
--- a/frappe/database/init.py
+++ b/frappe/database/init.py
@ -23,17 +23,17 @@ def setup_database(force, verbose=None, no_mariadb_socket=False):
 		)


-def bootstrap_database(db_name, verbose=None, source_sql=None):
+def bootstrap_database(verbose=None, source_sql=None):
 	import frappe

 	if frappe.conf.db_type == "postgres":
 		import frappe.database.postgres.setup_db

-		return frappe.database.postgres.setup_db.bootstrap_database(db_name, verbose, source_sql)
+		return frappe.database.postgres.setup_db.bootstrap_database(verbose, source_sql)
 	else:
 		import frappe.database.mariadb.setup_db

-		return frappe.database.mariadb.setup_db.bootstrap_database(db_name, verbose, source_sql)
+		return frappe.database.mariadb.setup_db.bootstrap_database(verbose, source_sql)


 def drop_user_and_database(db_name, db_user):
@ -75,12 +75,7 @@ def get_command(
 		else:
 			bin, bin_name = which("psql"), "psql"

-		host = frappe.utils.esc(host, "$ ")
-		user = frappe.utils.esc(user, "$ ")
-		db_name = frappe.utils.esc(db_name, "$ ")
-
 		if password:
-			password = frappe.utils.esc(password, "$ ")
 			conn_string = f"postgresql://{user}:{password}@{host}:{port}/{db_name}"
 		else:
 			conn_string = f"postgresql://{user}@{host}:{port}/{db_name}"
@ -96,10 +91,6 @@ def get_command(
 		else:
 			bin, bin_name = which("mariadb") or which("mysql"), "mariadb"

-		host = frappe.utils.esc(host, "$ ")
-		user = frappe.utils.esc(user, "$ ")
-		db_name = frappe.utils.esc(db_name, "$ ")
-
 		command = [
 			f"--user={user}",
 			f"--host={host}",
@ -107,7 +98,6 @@ def get_command(
 		]

 		if password:
-			password = frappe.utils.esc(password, "$ ")
 			command.append(f"--password={password}")

 		if dump:
--- a/frappe/database/mariadb/database.py
+++ b/frappe/database/mariadb/database.py
@ -383,6 +383,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 				WHERE Column_name = "{fieldname}"
 					AND Seq_in_index = 1
 					AND Non_unique={int(not unique)}
+					AND Index_type != 'FULLTEXT'
 				""",
 			as_dict=True,
 		)
--- a/frappe/database/mariadb/setup_db.py
+++ b/frappe/database/mariadb/setup_db.py
@ -67,17 +67,17 @@ def drop_user_and_database(
 	dbman.delete_user(db_user)


-def bootstrap_database(db_name, verbose, source_sql=None):
+def bootstrap_database(verbose, source_sql=None):
 	import sys

-	frappe.connect(db_name=db_name)
+	frappe.connect()
 	if not check_database_settings():
 		print("Database settings do not match expected values; stopping database setup.")
 		sys.exit(1)

 	import_db_from_sql(source_sql, verbose)
-	frappe.connect(db_name=db_name)

+	frappe.connect()
 	if "tabDefaultValue" not in frappe.db.get_tables(cached=False):
 		from click import secho

--- a/frappe/database/postgres/setup_db.py
+++ b/frappe/database/postgres/setup_db.py
@ -29,11 +29,12 @@ def setup_database():
 	root_conn.close()


-def bootstrap_database(db_name, verbose, source_sql=None):
-	frappe.connect(db_name=db_name)
-	import_db_from_sql(source_sql, verbose)
-	frappe.connect(db_name=db_name)
+def bootstrap_database(verbose, source_sql=None):

+	frappe.connect()
+	import_db_from_sql(source_sql, verbose)
+
+	frappe.connect()
 	if "tabDefaultValue" not in frappe.db.get_tables():
 		import sys

--- a/frappe/desk/doctype/event/event.js
+++ b/frappe/desk/doctype/event/event.js
@ -44,9 +44,13 @@ frappe.ui.form.on("Event", {

 		const [ends_on_date] = frm.doc.ends_on
 			? frm.doc.ends_on.split(" ")
-			: frm.doc.starts_on.split(" ");
+			: frm.doc.starts_on?.split(" ") || [];

-		if (frm.doc.google_meet_link && frappe.datetime.now_date() <= ends_on_date) {
+		if (
+			ends_on_date &&
+			frm.doc.google_meet_link &&
+			frappe.datetime.now_date() <= ends_on_date
+		) {
 			frm.dashboard.set_headline(
 				__("Join video conference with {0}", [
 					`<a target='_blank' href='${frm.doc.google_meet_link}'>Google Meet</a>`,
--- a/frappe/desk/doctype/note/note.py
+++ b/frappe/desk/doctype/note/note.py
@ -28,6 +28,9 @@ class Note(Document):
 			# expire this notification in a week (default)
 			self.expire_notification_on = frappe.utils.add_days(self.creation, 7)

+		if not self.public and self.notify_on_login:
+			self.notify_on_login = 0
+
 		if not self.content:
 			self.content = "<span></span>"

--- a/frappe/desk/form/test_form.py
+++ b/frappe/desk/form/test_form.py
@ -11,10 +11,3 @@ class TestForm(FrappeTestCase):
 		results = get_linked_docs("Role", "System Manager", linkinfo=get_linked_doctypes("Role"))
 		self.assertTrue("User" in results)
 		self.assertTrue("DocType" in results)
-
-
-if __name__ == "__main__":
-	import unittest
-
-	frappe.connect()
-	unittest.main()
--- a/frappe/desk/notifications.py
+++ b/frappe/desk/notifications.py
@ -25,7 +25,7 @@ def get_notifications():
 		"open_count_doctype": {},
 		"targets": {},
 	}
-	if frappe.flags.in_install or not frappe.db.get_single_value("System Settings", "setup_complete"):
+	if frappe.flags.in_install or not frappe.get_system_settings("setup_complete"):
 		return out

 	config = get_notification_config()
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@ -291,9 +291,9 @@ def get_prepared_report_result(report, filters, dn="", user=None):
 		try:
 			if data := json.loads(doc.get_prepared_data().decode("utf-8")):
 				report_data = get_report_data(doc, data)
-		except Exception:
+		except Exception as e:
 			doc.log_error("Prepared report render failed")
-			frappe.msgprint(_("Prepared report render failed"))
+			frappe.msgprint(_("Prepared report render failed") + f": {str(e)}")
 			doc = None

 	return report_data | {"prepared_report": True, "doc": doc}
--- a/frappe/email/doctype/auto_email_report/auto_email_report.json
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.json
@ -25,6 +25,7 @@
  "to_date_field",
  "column_break_17",
  "dynamic_date_period",
+  "use_first_day_of_period",
  "email_settings",
  "email_to",
  "day_of_week",
@ -87,6 +88,7 @@
  },
  {
   "default": "100",
+   "depends_on": "eval:doc.report_type=='Report Builder'",
   "fieldname": "no_of_rows",
   "fieldtype": "Int",
   "label": "No of Rows (Max 500)"
@ -207,10 +209,18 @@
   "fieldtype": "Link",
   "label": "Sender",
   "options": "Email Account"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval: doc.dynamic_date_period != 'Daily'",
+   "description": "To begin the date range at the start of the chosen period. For example, if 'Year' is selected as the period, the report will start from January 1st of the current year.",
+   "fieldname": "use_first_day_of_period",
+   "fieldtype": "Check",
+   "label": "Use First Day of Period"
  }
 ],
 "links": [],
- "modified": "2022-09-08 15:31:55.031023",
+ "modified": "2024-02-04 13:31:08.624648",
 "modified_by": "Administrator",
 "module": "Email",
 "name": "Auto Email Report",
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@ -2,6 +2,7 @@
 # License: MIT. See LICENSE

 import calendar
+import datetime
 from datetime import timedelta
 from email.utils import formataddr

@ -14,8 +15,13 @@ from frappe.utils import (
 	add_to_date,
 	cint,
 	format_time,
+	get_first_day,
+	get_first_day_of_week,
 	get_link_to_form,
+	get_quarter_start,
 	get_url_to_report,
+	get_year_start,
+	getdate,
 	global_date_format,
 	now,
 	now_datetime,
@ -57,8 +63,10 @@ class AutoEmailReport(Document):
 		send_if_data: DF.Check
 		sender: DF.Link | None
 		to_date_field: DF.Literal
+		use_first_day_of_period: DF.Check
 		user: DF.Link
 	# end: auto-generated types
+
 	def autoname(self):
 		self.name = _(self.report)
 		if frappe.db.exists("Auto Email Report", self.name):
@ -92,7 +100,7 @@ class AutoEmailReport(Document):

 		max_reports_per_user = (
 			cint(frappe.local.conf.max_reports_per_user)  # kept for backward compatibilty
-			or cint(frappe.db.get_single_value("System Settings", "max_auto_email_report_per_user"))
+			or cint(frappe.get_system_settings("max_auto_email_report_per_user"))
 			or 20
 		)

@ -207,17 +215,37 @@ class AutoEmailReport(Document):
 		self.filters = frappe.parse_json(self.filters)

 		to_date = today()
-		from_date_value = {
-			"Daily": ("days", -1),
-			"Weekly": ("weeks", -1),
-			"Monthly": ("months", -1),
-			"Quarterly": ("months", -3),
-			"Half Yearly": ("months", -6),
-			"Yearly": ("years", -1),
-		}[self.dynamic_date_period]

-		from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
+		if self.use_first_day_of_period:
+			from_date = to_date
+			if self.dynamic_date_period == "Daily":
+				from_date = add_to_date(to_date, days=-1)
+			elif self.dynamic_date_period == "Weekly":
+				from_date = get_first_day_of_week(from_date)
+			elif self.dynamic_date_period == "Monthly":
+				from_date = get_first_day(from_date)
+			elif self.dynamic_date_period == "Quarterly":
+				from_date = get_quarter_start(from_date)
+			elif self.dynamic_date_period == "Half Yearly":
+				from_date = get_half_year_start(from_date)
+			elif self.dynamic_date_period == "Yearly":
+				from_date = get_year_start(from_date)

+			self.set_date_filters(from_date, to_date)
+		else:
+			from_date_value = {
+				"Daily": ("days", -1),
+				"Weekly": ("weeks", -1),
+				"Monthly": ("months", -1),
+				"Quarterly": ("months", -3),
+				"Half Yearly": ("months", -6),
+				"Yearly": ("years", -1),
+			}[self.dynamic_date_period]
+
+			from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
+			self.set_date_filters(from_date, to_date)
+
+	def set_date_filters(self, from_date, to_date):
 		self.filters[self.from_date_field] = from_date
 		self.filters[self.to_date_field] = to_date

@ -332,3 +360,23 @@ def update_field_types(columns):
 			col.fieldtype = "Data"
 			col.options = ""
 	return columns
+
+
+DATE_FORMAT = "%Y-%m-%d"
+
+
+def get_half_year_start(as_str=False):
+	"""
+	Returns the first day of the current half-year based on the current date.
+	"""
+	today_date = getdate(today())
+
+	half_year = 1 if today_date.month <= 6 else 2
+
+	year = today_date.year if half_year == 1 else today_date.year + 1
+	month = 1 if half_year == 1 else 7
+	day = 1
+
+	result_date = datetime.date(year, month, day)
+
+	return result_date if not as_str else result_date.strftime(DATE_FORMAT)
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@ -131,12 +131,12 @@ class EmailQueue(Document):
 	def attachments_list(self):
 		return json.loads(self.attachments) if self.attachments else []

-	def get_email_account(self):
+	def get_email_account(self, raise_error=False):
 		if self.email_account:
 			return frappe.get_cached_doc("Email Account", self.email_account)

 		return EmailAccount.find_outgoing(
-			match_by_email=self.sender, match_by_doctype=self.reference_doctype
+			match_by_email=self.sender, match_by_doctype=self.reference_doctype, _raise_error=raise_error
 		)

 	def is_to_be_sent(self):
@ -158,6 +158,7 @@ class EmailQueue(Document):
 			return

 		with SendMailContext(self, smtp_server_instance) as ctx:
+			ctx.fetch_smtp_server()
 			message = None
 			for recipient in self.recipients:
 				if recipient.is_mail_sent():
@ -233,14 +234,16 @@ class SendMailContext:
 		smtp_server_instance: SMTPServer = None,
 	):
 		self.queue_doc: EmailQueue = queue_doc
-		self.email_account_doc = queue_doc.get_email_account()
-
-		self.smtp_server: SMTPServer = smtp_server_instance or self.email_account_doc.get_smtp_server()
-
+		self.smtp_server: SMTPServer = smtp_server_instance
 		self.sent_to_atleast_one_recipient = any(
 			rec.recipient for rec in self.queue_doc.recipients if rec.is_mail_sent()
 		)

+	def fetch_smtp_server(self):
+		self.email_account_doc = self.queue_doc.get_email_account(raise_error=True)
+		if not self.smtp_server:
+			self.smtp_server = self.email_account_doc.get_smtp_server()
+
 	def __enter__(self):
 		self.queue_doc.update_status(status="Sending", commit=True)
 		return self
@ -733,7 +736,7 @@ class QueueBuilder:
 			recipients = list(set([r] + self.final_cc() + self.bcc))
 			q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True)
 			if not smtp_server_instance:
-				email_account = q.get_email_account()
+				email_account = q.get_email_account(raise_error=True)
 				smtp_server_instance = email_account.get_smtp_server()

 			with suppress(Exception):
--- a/frappe/email/doctype/newsletter/newsletter.json
+++ b/frappe/email/doctype/newsletter/newsletter.json
@ -3,7 +3,7 @@
 "allow_guest_to_view": 1,
 "allow_rename": 1,
 "creation": "2013-01-10 16:34:31",
- "description": "Create and Send Newsletters",
+ "description": "Create and send emails to a specific group of subscribers periodically.",
 "doctype": "DocType",
 "document_type": "Other",
 "engine": "InnoDB",
@ -244,8 +244,7 @@
   "fieldname": "campaign",
   "fieldtype": "Link",
   "label": "Campaign",
-   "options": "Marketing Campaign",
-   "reqd": 0
+   "options": "Marketing Campaign"
  }
 ],
 "has_web_view": 1,
@ -254,7 +253,7 @@
 "index_web_pages_for_search": 1,
 "is_published_field": "published",
 "links": [],
- "modified": "2023-12-29 18:04:13.270523",
+ "modified": "2024-01-30 14:05:50.645802",
 "modified_by": "Administrator",
 "module": "Email",
 "name": "Newsletter",
--- a/frappe/email/receive.py
+++ b/frappe/email/receive.py
@ -673,7 +673,7 @@ class InboundMail(Email):
 		content = self.content
 		for file in attachments:
 			if file.name in self.cid_map and self.cid_map[file.name]:
-				content = content.replace(f"cid:{self.cid_map[file.name]}", file.file_url)
+				content = content.replace(f"cid:{self.cid_map[file.name]}", file.unique_url)
 		return content

 	def is_notification(self):
--- a/frappe/geo/doctype/currency/currency.json
+++ b/frappe/geo/doctype/currency/currency.json
@ -4,7 +4,7 @@
 "allow_rename": 1,
 "autoname": "field:currency_name",
 "creation": "2013-01-28 10:06:02",
- "description": "**Currency** Master",
+ "description": "Currency list stores the currency value, its symbol and fraction unit",
 "doctype": "DocType",
 "document_type": "Setup",
 "engine": "InnoDB",
@ -82,7 +82,7 @@
 "idx": 1,
 "index_web_pages_for_search": 1,
 "links": [],
- "modified": "2024-01-17 15:37:31.605278",
+ "modified": "2024-01-30 13:18:12.053557",
 "modified_by": "Administrator",
 "module": "Geo",
 "name": "Currency",
--- a/frappe/gettext/extractors/html_template.py
+++ b/frappe/gettext/extractors/html_template.py
@ -0,0 +1,26 @@
+from jinja2.ext import babel_extract
+
+from .utils import extract_messages_from_code
+
+
+def extract(*args, **kwargs):
+	"""Extract messages from Jinja and JS microtemplates.
+
+	Reuse the babel_extract function from jinja2.ext, but handle our own implementation of `_()`.
+	To handle JS microtemplates, parse all code again using regex."""
+	fileobj = args[0] or kwargs["fileobj"]
+	print(fileobj.name)
+	code = fileobj.read().decode("utf-8")
+
+	for lineno, funcname, messages, comments in babel_extract(*args, **kwargs):
+		if funcname == "_" and isinstance(messages, tuple) and len(messages) > 1:
+			funcname = "pgettext"
+			messages = (messages[-1], messages[0])  # (context, message)
+
+		yield lineno, funcname, messages, comments
+
+	for lineno, message, context in extract_messages_from_code(code):
+		if context:
+			yield lineno, "pgettext", (context, message), []
+		else:
+			yield lineno, "_", message, []
--- a/frappe/gettext/extractors/jinja2.py
+++ b/frappe/gettext/extractors/jinja2.py
@ -1,11 +0,0 @@
-from jinja2.ext import babel_extract
-
-
-def extract(*args, **kwargs):
-	"""Reuse the babel_extract function from jinja2.ext, but handle our own implementation of `_()`"""
-	for lineno, funcname, messages, comments in babel_extract(*args, **kwargs):
-		if funcname == "_" and isinstance(messages, tuple) and len(messages) > 1:
-			funcname = "pgettext"
-			messages = (messages[-1], messages[0])  # (context, message)
-
-		yield lineno, funcname, messages, comments
--- a/frappe/gettext/extractors/utils.py
+++ b/frappe/gettext/extractors/utils.py
@ -0,0 +1,81 @@
+import re
+
+import frappe
+
+TRANSLATE_PATTERN = re.compile(
+	r"_\(\s*"  # starts with literal `_(`, ignore following whitespace/newlines
+	# BEGIN: message search
+	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
+	r"(?P<message>((?!\1).)*)"  # Keep matching until string closing identifier is met which is same as 1st capture group
+	r"\1"  # match exact string closing identifier
+	# END: message search
+	# BEGIN: python context search
+	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
+	r"([\"'])"  # start of context string identifier; 5th capture group
+	r"(?P<py_context>((?!\5).)*)"  # capture context string till closing id is found
+	r"\5"  # match context string closure
+	r")?"  # match 0 or 1 context strings
+	# END: python context search
+	# BEGIN: JS context search
+	r"(\s*,\s*(.)*?\s*(,\s*"  # skip message format replacements: ["format", ...] | null | []
+	r"([\"'])"  # start of context string; 11th capture group
+	r"(?P<js_context>((?!\11).)*)"  # capture context string till closing id is found
+	r"\11"  # match context string closure
+	r")*"
+	r")*"  # match one or more context string
+	# END: JS context search
+	r"\s*\)"  # Closing function call ignore leading whitespace/newlines
+)
+
+
+def extract_messages_from_code(code):
+	"""
+	Extracts translatable strings from a code file
+	:param code: code from which translatable files are to be extracted
+	"""
+	from jinja2 import TemplateError
+
+	from frappe.model.utils import InvalidIncludePath, render_include
+
+	try:
+		code = frappe.as_unicode(render_include(code))
+
+	# Exception will occur when it encounters John Resig's microtemplating code
+	except (TemplateError, ImportError, InvalidIncludePath, OSError) as e:
+		if isinstance(e, InvalidIncludePath) and hasattr(frappe.local, "message_log"):
+			frappe.clear_last_message()
+
+	messages = []
+
+	for m in TRANSLATE_PATTERN.finditer(code):
+		message = m.group("message")
+		context = m.group("py_context") or m.group("js_context")
+		pos = m.start()
+
+		if is_translatable(message):
+			messages.append([pos, message, context])
+
+	return add_line_number(messages, code)
+
+
+def is_translatable(m):
+	return bool(
+		re.search("[a-zA-Z]", m)
+		and not m.startswith("fa fa-")
+		and not m.endswith("px")
+		and not m.startswith("eval:")
+	)
+
+
+def add_line_number(messages, code):
+	ret = []
+	messages = sorted(messages, key=lambda x: x[0])
+	newlines = [m.start() for m in re.compile(r"\n").finditer(code)]
+	line = 1
+	newline_i = 0
+	for pos, message, context in messages:
+		while newline_i < len(newlines) and pos > newlines[newline_i]:
+			line += 1
+			newline_i += 1
+		ret.append([line, message, context])
+	return ret
--- a/frappe/handler.py
+++ b/frappe/handler.py
@ -250,7 +250,7 @@ def check_write_permission(doctype: str = None, name: str = None):
 	if doctype and name:
 		try:
 			doc = frappe.get_doc(doctype, name)
-			doc.has_permission("write")
+			doc.check_permission("write")
 		except frappe.DoesNotExistError:
 			# doc has not been inserted yet, name is set to "new-some-doctype"
 			check_doctype = True
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@ -532,15 +532,15 @@ standard_help_items = [

 # log doctype cleanups to automatically add in log settings
 default_log_clearing_doctypes = {
-	"Error Log": 30,
-	"Activity Log": 90,
+	"Error Log": 14,
 	"Email Queue": 30,
-	"Scheduled Job Log": 90,
-	"Route History": 90,
-	"Submission Queue": 30,
-	"Prepared Report": 30,
+	"Scheduled Job Log": 7,
+	"Submission Queue": 7,
+	"Prepared Report": 14,
 	"Webhook Request Log": 30,
-	"Integration Request": 90,
 	"Unhandled Email": 30,
 	"Reminder": 30,
+	"Integration Request": 90,
+	"Activity Log": 90,
+	"Route History": 90,
 }
--- a/frappe/installer.py
+++ b/frappe/installer.py
@ -20,9 +20,10 @@ from frappe.utils.dashboard import sync_dashboards
 from frappe.utils.synchronization import filelock


-def _is_scheduler_enabled() -> bool:
+def _is_scheduler_enabled(site) -> bool:
 	enable_scheduler = False
 	try:
+		frappe.init(site=site)
 		frappe.connect()
 		enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler"))
 	except Exception:
@ -78,7 +79,7 @@ def _new_site(

 	try:
 		# enable scheduler post install?
-		enable_scheduler = _is_scheduler_enabled()
+		enable_scheduler = _is_scheduler_enabled(site)
 	except Exception:
 		enable_scheduler = False

@ -170,7 +171,6 @@ def install_db(
 		setup_database(force, verbose, no_mariadb_socket)

 	bootstrap_database(
-		db_name=frappe.conf.db_name,
 		verbose=verbose,
 		source_sql=source_sql,
 	)
--- a/frappe/integrations/doctype/google_calendar/google_calendar.py
+++ b/frappe/integrations/doctype/google_calendar/google_calendar.py
@ -239,7 +239,7 @@ def check_google_calendar(account, google_calendar):
 			# If no Calendar ID create a new Calendar
 			calendar = {
 				"summary": account.calendar_name,
-				"timeZone": frappe.db.get_single_value("System Settings", "time_zone"),
+				"timeZone": frappe.get_system_settings("time_zone"),
 			}
 			created_calendar = google_calendar.calendars().insert(body=calendar).execute()
 			frappe.db.set_value(
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@ -41,9 +41,6 @@ def send_email(success, service_name, doctype, email_field, error_status=None):


 def get_recipients(doctype, email_field):
-	if not frappe.db:
-		frappe.connect()
-
 	return split_emails(frappe.db.get_value(doctype, None, email_field))


--- a/frappe/locale/de.po
+++ b/frappe/locale/de.po
--- a/frappe/locale/main.pot
+++ b/frappe/locale/main.pot
--- a/frappe/model/init.py
+++ b/frappe/model/init.py
@ -134,22 +134,22 @@ log_types = (
 )

 std_fields = [
-	{"fieldname": "name", "fieldtype": "Link", "label": _lt("ID")},
-	{"fieldname": "owner", "fieldtype": "Link", "label": _lt("Created By"), "options": "User"},
-	{"fieldname": "idx", "fieldtype": "Int", "label": _lt("Index")},
-	{"fieldname": "creation", "fieldtype": "Datetime", "label": _lt("Created On")},
-	{"fieldname": "modified", "fieldtype": "Datetime", "label": _lt("Last Updated On")},
+	{"fieldname": "name", "fieldtype": "Link", "label": "ID"},
+	{"fieldname": "owner", "fieldtype": "Link", "label": "Created By", "options": "User"},
+	{"fieldname": "idx", "fieldtype": "Int", "label": "Index"},
+	{"fieldname": "creation", "fieldtype": "Datetime", "label": "Created On"},
+	{"fieldname": "modified", "fieldtype": "Datetime", "label": "Last Updated On"},
 	{
 		"fieldname": "modified_by",
 		"fieldtype": "Link",
-		"label": _lt("Last Updated By"),
+		"label": "Last Updated By",
 		"options": "User",
 	},
-	{"fieldname": "_user_tags", "fieldtype": "Data", "label": _lt("Tags")},
-	{"fieldname": "_liked_by", "fieldtype": "Data", "label": _lt("Liked By")},
-	{"fieldname": "_comments", "fieldtype": "Text", "label": _lt("Comments")},
-	{"fieldname": "_assign", "fieldtype": "Text", "label": _lt("Assigned To")},
-	{"fieldname": "docstatus", "fieldtype": "Int", "label": _lt("Document Status")},
+	{"fieldname": "_user_tags", "fieldtype": "Data", "label": "Tags"},
+	{"fieldname": "_liked_by", "fieldtype": "Data", "label": "Liked By"},
+	{"fieldname": "_comments", "fieldtype": "Text", "label": "Comments"},
+	{"fieldname": "_assign", "fieldtype": "Text", "label": "Assigned To"},
+	{"fieldname": "docstatus", "fieldtype": "Int", "label": "Document Status"},
 ]


@ -230,6 +230,9 @@ def get_permitted_fields(
 	if permission_type is None:
 		permission_type = "select" if frappe.only_has_select_perm(doctype, user=user) else "read"

+	meta_fields = meta.default_fields.copy()
+	optional_meta_fields = [x for x in optional_fields if x in valid_columns]
+
 	if permitted_fields := meta.get_permitted_fieldnames(
 		parenttype=parenttype,
 		user=user,
@ -239,15 +242,12 @@ def get_permitted_fields(
 		if permission_type == "select":
 			return permitted_fields

-		meta_fields = meta.default_fields.copy()
-		optional_meta_fields = [x for x in optional_fields if x in valid_columns]
-
 		if meta.istable:
 			meta_fields.extend(child_table_fields)

 		return meta_fields + permitted_fields + optional_meta_fields

-	return []
+	return meta_fields + optional_meta_fields


 def is_default_field(fieldname: str) -> bool:
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@ -217,6 +217,10 @@ class DatabaseQuery:
 		args = self.prepare_args()
 		args.limit = self.add_limit()

+		if not args.fields:
+			# apply_fieldlevel_read_permissions has likely removed ALL the fields that user asked for
+			return []
+
 		if args.conditions:
 			args.conditions = "where " + args.conditions

--- a/frappe/model/delete_doc.py
+++ b/frappe/model/delete_doc.py
@ -133,6 +133,7 @@ def delete_doc(
 					doctype=doc.doctype,
 					name=doc.name,
 					now=frappe.flags.in_test,
+					enqueue_after_commit=True,
 				)

 		# clear cache for Document
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@ -679,23 +679,15 @@ class Document(BaseDocument):
 		return same

 	def apply_fieldlevel_read_permissions(self):
-		"""Remove values the user is not allowed to read (called when loading in desk)"""
-
+		"""Remove values the user is not allowed to read."""
 		if frappe.session.user == "Administrator":
 			return

-		has_higher_permlevel = False
-
 		all_fields = self.meta.fields.copy()
 		for table_field in self.meta.get_table_fields():
 			all_fields += frappe.get_meta(table_field.options).fields or []

-		for df in all_fields:
-			if df.permlevel > 0:
-				has_higher_permlevel = True
-				break
-
-		if not has_higher_permlevel:
+		if all(df.permlevel == 0 for df in all_fields):
 			return

 		has_access_to = self.get_permlevel_access("read")
--- a/frappe/model/meta.py
+++ b/frappe/model/meta.py
@ -596,6 +596,10 @@ class Meta(Document):
 			self.get_permlevel_access(permission_type=permission_type, parenttype=parenttype, user=user)
 		)

+		if 0 not in permlevel_access and permission_type in ("read", "select"):
+			if frappe.share.get_shared(self.name, user, rights=[permission_type], limit=1):
+				permlevel_access.add(0)
+
 		permitted_fieldnames.extend(
 			df.fieldname
 			for df in self.get_fieldnames_with_value(
--- a/frappe/model/utils/init.py
+++ b/frappe/model/utils/init.py
@ -66,7 +66,7 @@ def render_include(content):
 		if "{% include" in content:
 			paths = INCLUDE_DIRECTIVE_PATTERN.findall(content)
 			if not paths:
-				frappe.throw(_("Invalid include path"), InvalidIncludePath)
+				raise InvalidIncludePath

 			for path in paths:
 				app, app_path = path.split("/", 1)
--- a/frappe/printing/doctype/letter_head/letter_head.js
+++ b/frappe/printing/doctype/letter_head/letter_head.js
@ -2,7 +2,60 @@
 // For license information, please see license.txt

 frappe.ui.form.on("Letter Head", {
+	setup(frm) {
+		frm.get_field("instructions").html(INSTRUCTIONS);
+	},
+
 	refresh: function (frm) {
 		frm.flag_public_attachments = true;
 	},
+
+	validate: (frm) => {
+		["header_script", "footer_script"].forEach((field) => {
+			if (!frm.doc[field]) return;
+
+			try {
+				eval(frm.doc[field]);
+			} catch (e) {
+				frappe.throw({
+					title: __("Error in Header/Footer Script"),
+					indicator: "orange",
+					message: '<pre class="small"><code>' + e.stack + "</code></pre>",
+				});
+			}
+		});
+	},
 });
+
+const INSTRUCTIONS = `<h4>${__("Letter Head Scripts")}</h4>
+<p>${__("Header/Footer scripts can be used to add dynamic behaviours.")}</p>
+<pre>
+<code>
+// ${__(
+	"The following Header Script will add the current date to an element in 'Header HTML' with class 'header-content'"
+)}
+var el = document.getElementsByClassName("header-content");
+if (el.length > 0) {
+	el[0].textContent += " " + new Date().toGMTString();
+}
+</code>
+</pre>
+<p>${__("You can also access wkhtmltopdf variables (valid only in PDF print):")}</p>
+<pre>
+<code>
+// ${__("Get Header and Footer wkhtmltopdf variables")}
+// ${__("Snippet and more variables:  {0}", ["https://wkhtmltopdf.org/usage/wkhtmltopdf.txt"])}
+var vars = {};
+var query_strings_from_url = document.location.search.substring(1).split('&');
+for (var query_string in query_strings_from_url) {
+	if (query_strings_from_url.hasOwnProperty(query_string)) {
+		var temp_var = query_strings_from_url[query_string].split('=', 2);
+		vars[temp_var[0]] = decodeURI(temp_var[1]);
+	}
+}
+var el = document.getElementsByClassName("header-content");
+if (el.length > 0 && vars["page"] == 1) {
+	el[0].textContent += " : " + vars["date"];
+}
+</code>
+</pre>`;
--- a/frappe/printing/doctype/letter_head/letter_head.json
+++ b/frappe/printing/doctype/letter_head/letter_head.json
@ -26,7 +26,11 @@
  "footer_image",
  "footer_image_height",
  "footer_image_width",
-  "footer_align"
+  "footer_align",
+  "scripts_section",
+  "header_script",
+  "footer_script",
+  "instructions"
 ],
 "fields": [
  {
@ -162,13 +166,40 @@
   "fieldtype": "Select",
   "label": "Footer Based On",
   "options": "Image\nHTML"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && doc.source==='HTML'",
+   "fieldname": "header_script",
+   "fieldtype": "Code",
+   "label": "Header Script",
+   "options": "Javascript"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && doc.footer_source==='HTML'",
+   "fieldname": "footer_script",
+   "fieldtype": "Code",
+   "label": "Footer Script",
+   "options": "Javascript"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "eval: doc.header_script || doc.footer_script",
+   "fieldname": "scripts_section",
+   "fieldtype": "Section Break",
+   "label": "Scripts"
+  },
+  {
+   "fieldname": "instructions",
+   "fieldtype": "HTML",
+   "label": "Instructions",
+   "read_only": 1
  }
 ],
 "icon": "fa fa-font",
 "idx": 1,
 "links": [],
 "max_attachments": 3,
- "modified": "2023-12-08 15:52:37.525003",
+ "modified": "2023-12-21 16:19:37.525003",
 "modified_by": "Administrator",
 "module": "Printing",
 "name": "Letter Head",
--- a/frappe/public/js/form_builder/store.js
+++ b/frappe/public/js/form_builder/store.js
@ -85,8 +85,13 @@ export const useStore = defineStore("form-builder-store", () => {

 	async function fetch() {
 		doc.value = frm.value.doc;
-		if (doctype.value.startsWith("new-doctype-") && !doc.value.fields) {
-			doc.value.fields = [get_df("Data", "", __("Title"))];
+		if (doctype.value.startsWith("new-doctype-") && !doc.value.fields?.length) {
+			frappe.model.with_doctype("DocType").then(() => {
+				frappe.listview_settings["DocType"].new_doctype_dialog();
+			});
+			// redirect to /doctype
+			frappe.set_route("List", "DocType");
+			return;
 		}

 		if (!get_docfields.value.length) {
--- a/frappe/public/js/frappe/assets.js
+++ b/frappe/public/js/frappe/assets.js
@ -19,11 +19,22 @@ frappe.require = function (items, callback) {
 	});
 };

-frappe.assets = {
-	check: function () {
+class AssetManager {
+	constructor() {
+		this._executed = [];
+		this._handlers = {
+			js: function (txt) {
+				frappe.dom.eval(txt);
+			},
+			css: function (txt) {
+				frappe.dom.set_style(txt);
+			},
+		};
+	}
+	check() {
 		// if version is different then clear localstorage
 		if (window._version_number != localStorage.getItem("_version_number")) {
-			frappe.assets.clear_local_storage();
+			this.clear_local_storage();
 			console.log("Cleared App Cache.");
 		}

@ -33,160 +44,79 @@ frappe.assets = {
 			// Evict cache if page is reloaded within 10 seconds. Which could be user trying to
 			// refresh if things feel broken.
 			if ((not_updated_since < 5000 && is_reload()) || not_updated_since > 2 * 86400000) {
-				frappe.assets.clear_local_storage();
+				this.clear_local_storage();
 			}
 		} else {
-			frappe.assets.clear_local_storage();
+			this.clear_local_storage();
 		}

-		frappe.assets.init_local_storage();
-	},
+		this.init_local_storage();
+	}

-	init_local_storage: function () {
+	init_local_storage() {
 		localStorage._last_load = new Date();
 		localStorage._version_number = window._version_number;
 		if (frappe.boot) localStorage.metadata_version = frappe.boot.metadata_version;
-	},
+	}

-	clear_local_storage: function () {
-		$.each(
-			["_last_load", "_version_number", "metadata_version", "page_info", "last_visited"],
-			function (i, key) {
-				localStorage.removeItem(key);
-			}
+	clear_local_storage() {
+		["_last_load", "_version_number", "metadata_version", "page_info", "last_visited"].forEach(
+			(key) => localStorage.removeItem(key)
 		);

 		// clear assets
-		for (var key in localStorage) {
+		for (let key in localStorage) {
 			if (
-				key.indexOf("desk_assets:") === 0 ||
-				key.indexOf("_page:") === 0 ||
-				key.indexOf("_doctype:") === 0 ||
-				key.indexOf("preferred_breadcrumbs:") === 0
+				key.startsWith("_page:") ||
+				key.startsWith("_doctype:") ||
+				key.startsWith("preferred_breadcrumbs:")
 			) {
 				localStorage.removeItem(key);
 			}
 		}
 		console.log("localStorage cleared");
-	},
+	}

-	// keep track of executed assets
-	executed_: [],
+	eval_assets(path, content) {
+		if (!this._executed.includes(path)) {
+			this._handlers[this.extn(path)](content);
+			this._executed.push(path);
+		}
+	}

-	// pass on to the handler to set
-	execute: function (items, callback) {
-		var to_fetch = [];
-		for (var i = 0, l = items.length; i < l; i++) {
-			if (!frappe.assets.exists(items[i])) {
-				to_fetch.push(items[i]);
-			}
-		}
-		if (to_fetch.length) {
-			frappe.assets.fetch(to_fetch, function () {
-				frappe.assets.eval_assets(items, callback);
-			});
-		} else {
-			frappe.assets.eval_assets(items, callback);
-		}
-	},
-
-	eval_assets: function (items, callback) {
-		for (var i = 0, l = items.length; i < l; i++) {
-			// execute js/css if not already.
-			var path = items[i];
-			if (frappe.assets.executed_.indexOf(path) === -1) {
-				// execute
-				frappe.assets.handler[frappe.assets.extn(path)](frappe.assets.get(path), path);
-				frappe.assets.executed_.push(path);
-			}
-		}
-		callback && callback();
-	},
-
-	// check if the asset exists in
-	// localstorage
-	exists: function (src) {
-		if (frappe.assets.executed_.indexOf(src) !== -1) {
-			return true;
-		}
-		if (frappe.boot.developer_mode) {
-			return false;
-		}
-		if (frappe.assets.get(src)) {
-			return true;
-		} else {
-			return false;
-		}
-	},
-
-	// load an asset via
-	fetch: function (items, callback) {
+	execute(items, callback) {
 		// this is virtual page load, only get the the source
-		// *without* the template
-
-		if (items.length === 0) {
-			callback();
-			return;
-		}
+		let me = this;

 		const version_string =
 			frappe.boot.developer_mode || window.dev_server ? Date.now() : window._version_number;

-		async function fetch_item(item) {
+		async function fetch_item(path) {
 			// Add the version to the URL to bust the cache for non-bundled assets
-			let url = new URL(item, window.location.origin);
+			let url = new URL(path, window.location.origin);

-			if (!item.includes(".bundle.") && !url.searchParams.get("v")) {
+			if (!path.includes(".bundle.") && !url.searchParams.get("v")) {
 				url.searchParams.append("v", version_string);
 			}
 			const response = await fetch(url.toString());
 			const body = await response.text();
-			frappe.assets.add(item, body);
+			me.eval_assets(path, body);
 		}

 		frappe.dom.freeze();
 		const fetch_promises = items.map(fetch_item);
 		Promise.all(fetch_promises).then(() => {
 			frappe.dom.unfreeze();
-			callback();
+			callback?.();
 		});
-	},
+	}

-	add: function (src, txt) {
-		if ("localStorage" in window) {
-			try {
-				frappe.assets.set(src, txt);
-			} catch (e) {
-				// if quota is exceeded, clear local storage and set item
-				frappe.assets.clear_local_storage();
-				frappe.assets.set(src, txt);
-			}
-		}
-	},
-
-	get: function (src) {
-		return localStorage.getItem("desk_assets:" + src);
-	},
-
-	set: function (src, txt) {
-		localStorage.setItem("desk_assets:" + src, txt);
-	},
-
-	extn: function (src) {
+	extn(src) {
 		if (src.indexOf("?") != -1) {
 			src = src.split("?").slice(-1)[0];
 		}
 		return src.split(".").slice(-1)[0];
-	},
-
-	handler: {
-		js: function (txt, src) {
-			frappe.dom.eval(txt);
-		},
-		css: function (txt, src) {
-			frappe.dom.set_style(txt);
-		},
-	},
+	}

 	bundled_asset(path, is_rtl = null) {
 		if (!path.startsWith("/assets") && path.includes(".bundle.")) {
@ -197,8 +127,8 @@ frappe.assets = {
 			return path;
 		}
 		return path;
-	},
-};
+	}
+}

 function is_reload() {
 	try {
@ -211,3 +141,5 @@ function is_reload() {
 		return true;
 	}
 }
+
+frappe.assets = new AssetManager();
--- a/frappe/public/js/frappe/build_events/build_events.bundle.js
+++ b/frappe/public/js/frappe/build_events/build_events.bundle.js
@ -17,7 +17,7 @@ frappe.realtime.on("build_event", (data) => {
 					if (parts.length === 2) {
 						let filename = parts[0].split("/").slice(-1)[0];

-						frappe.assets.executed_ = frappe.assets.executed_.filter(
+						frappe.assets._executed = frappe.assets._executed.filter(
 							(asset) => !asset.includes(`${filename}.bundle`)
 						);
 					}
--- a/frappe/public/js/frappe/data_import/import_preview.js
+++ b/frappe/public/js/frappe/data_import/import_preview.js
@ -34,7 +34,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
 					<div class="col-sm-12">
 						<div class="table-actions margin-bottom">
 						</div>
-						<div class="table-preview border"></div>
+						<div class="table-preview"></div>
 						<div class="table-message"></div>
 					</div>
 				</div>
--- a/frappe/public/js/frappe/form/controls/code.js
+++ b/frappe/public/js/frappe/form/controls/code.js
@ -153,6 +153,7 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex
 			JS: "ace/mode/javascript",
 			Python: "ace/mode/python",
 			Py: "ace/mode/python",
+			PythonExpression: "ace/mode/python",
 			HTML: "ace/mode/html",
 			CSS: "ace/mode/css",
 			Markdown: "ace/mode/markdown",
--- a/frappe/public/js/frappe/form/controls/link.js
+++ b/frappe/public/js/frappe/form/controls/link.js
@ -87,10 +87,10 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 		return this.is_translatable() ? __(value) : value;
 	}
 	is_translatable() {
-		return frappe.boot?.translated_doctypes || [].includes(this.get_options());
+		return (frappe.boot?.translated_doctypes || []).includes(this.get_options());
 	}
 	is_title_link() {
-		return frappe.boot?.link_title_doctypes || [].includes(this.get_options());
+		return (frappe.boot?.link_title_doctypes || []).includes(this.get_options());
 	}
 	async set_link_title(value) {
 		const doctype = this.get_options();
--- a/frappe/public/js/frappe/form/formatters.js
+++ b/frappe/public/js/frappe/form/formatters.js
@ -49,6 +49,10 @@ frappe.form.formatters = {
 		return __(frappe.form.formatters["Data"](value, df));
 	},
 	Float: function (value, docfield, options, doc) {
+		if (value === null) {
+			return "";
+		}
+
 		// don't allow 0 precision for Floats, hence or'ing with null
 		var precision =
 			docfield.precision ||
@ -73,12 +77,20 @@ frappe.form.formatters = {
 		}
 	},
 	Int: function (value, docfield, options) {
+		if (value === null) {
+			return "";
+		}
+
 		if (cstr(docfield.options).trim() === "File Size") {
 			return frappe.form.formatters.FileSize(value);
 		}
 		return frappe.form.formatters._right(value == null ? "" : cint(value), options);
 	},
 	Percent: function (value, docfield, options) {
+		if (value === null) {
+			return "";
+		}
+
 		const precision =
 			docfield.precision ||
 			cint(frappe.boot.sysdefaults && frappe.boot.sysdefaults.float_precision) ||
@ -105,6 +117,10 @@ frappe.form.formatters = {
 		</div>`;
 	},
 	Currency: function (value, docfield, options, doc) {
+		if (value === null) {
+			return "";
+		}
+
 		var currency = frappe.meta.get_field_currency(docfield, doc);

 		let precision;
--- a/frappe/public/js/frappe/form/grid_row.js
+++ b/frappe/public/js/frappe/form/grid_row.js
@ -461,6 +461,7 @@ export default class GridRow {
 					fieldname: "fields",
 					options: docfields,
 					columns: 2,
+					sort_options: false,
 				},
 			],
 		});
@ -495,12 +496,31 @@ export default class GridRow {

 		const show_field = (f) => always_allow.includes(f) || !blocked_fields.includes(f);

+		// First, add selected fields
+		selected_fields.forEach((selectedField) => {
+			const selectedColumn = this.docfields.find(
+				(column) => column.fieldname === selectedField
+			);
+			if (selectedColumn && !selectedColumn.hidden && show_field(selectedColumn.fieldtype)) {
+				fields.push({
+					label: selectedColumn.label,
+					value: selectedColumn.fieldname,
+					checked: true,
+				});
+			}
+		});
+
+		// Then, add the rest of the fields
 		this.docfields.forEach((column) => {
-			if (!column.hidden && show_field(column.fieldtype)) {
+			if (
+				!selected_fields.includes(column.fieldname) &&
+				!column.hidden &&
+				show_field(column.fieldtype)
+			) {
 				fields.push({
 					label: column.label,
 					value: column.fieldname,
-					checked: selected_fields ? selected_fields.includes(column.fieldname) : false,
+					checked: false,
 				});
 			}
 		});
--- a/frappe/public/js/frappe/list/base_list.js
+++ b/frappe/public/js/frappe/list/base_list.js
@ -132,7 +132,9 @@ frappe.views.BaseList = class BaseList {
 			frappe.meta.has_field(doctype, fieldname) ||
 			fieldname === "_seen";

-		if (!is_valid_field) {
+		let is_virtual = this.meta.fields.find((df) => df.fieldname == fieldname)?.is_virtual;
+
+		if (!is_valid_field || is_virtual) {
 			return;
 		}

--- a/frappe/public/js/frappe/list/list_view.js
+++ b/frappe/public/js/frappe/list/list_view.js
@ -374,7 +374,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 					if (frappe.has_indicator(this.doctype) && df.fieldname === "status") {
 						return false;
 					}
-					if (!df.in_list_view) {
+					if (!df.in_list_view || df.is_virtual) {
 						return false;
 					}
 					return df.fieldname !== this.meta.title_field;
@ -465,7 +465,10 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 			? __("No {0} found with matching filters. Clear filters to see all {0}.", [
 					__(this.doctype),
 			  ])
+			: this.meta.description
+			? __(this.meta.description)
 			: __("You haven't created a {0} yet", [__(this.doctype)]);
+
 		let new_button_label = has_filters_set
 			? __("Create a new {0}", [__(this.doctype)], "Create a new document from list view")
 			: __(
@ -1517,7 +1520,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 	}

 	avoid_realtime_update() {
-		if (this.filter_area.is_being_edited()) {
+		if (this.filter_area?.is_being_edited()) {
 			return true;
 		}
 		// this is set when a bulk operation is called from a list view which might update the list view
--- a/frappe/public/js/frappe/model/perm.js
+++ b/frappe/public/js/frappe/model/perm.js
@ -203,7 +203,7 @@ $.extend(frappe.perm, {

 		// permission
 		if (p) {
-			if (p.write && !df.disabled) {
+			if (p.write && !df.disabled && !df.is_virtual) {
 				status = "Write";
 			} else if (p.read) {
 				status = "Read";
--- a/frappe/public/js/frappe/utils/utils.js
+++ b/frappe/public/js/frappe/utils/utils.js
@ -1705,6 +1705,8 @@ Object.assign(frappe.utils, {
 					fieldname: "source",
 					label: __("Source"),
 					fieldtype: "Data",
+					reqd: 1,
+					description: "The referrer (e.g. google, newsletter)",
 					default: localStorage.getItem("tracker_url:source"),
 				},
 				{
@ -1719,25 +1721,35 @@ Object.assign(frappe.utils, {
 					fieldname: "medium",
 					label: __("Medium"),
 					fieldtype: "Data",
+					description: "Marketing medium (e.g. cpc, banner, email)",
 					default: localStorage.getItem("tracker_url:medium"),
 				},
+				{
+					fieldname: "content",
+					label: __("Content"),
+					fieldtype: "Data",
+					description: "Use to differentiate ad variants (e.g. A/B testing)",
+					default: localStorage.getItem("tracker_url:content"),
+				},
 			],
 			function (data) {
 				let url = data.url;
 				localStorage.setItem("tracker_url:url", data.url);

-				if (data.source) {
-					url += "?source=" + data.source;
-					localStorage.setItem("tracker_url:source", data.source);
-				}
+				url += "?utm_source=" + encodeURIComponent(data.source);
+				localStorage.setItem("tracker_url:source", data.source);
 				if (data.campaign) {
-					url += "&campaign=" + data.campaign;
+					url += "&utm_campaign=" + encodeURIComponent(data.campaign);
 					localStorage.setItem("tracker_url:campaign", data.campaign);
 				}
 				if (data.medium) {
-					url += "&medium=" + data.medium.toLowerCase();
+					url += "&utm_medium=" + encodeURIComponent(data.medium);
 					localStorage.setItem("tracker_url:medium", data.medium);
 				}
+				if (data.medium) {
+					url += "&utm_content=" + encodeURIComponent(data.content);
+					localStorage.setItem("tracker_url:content", data.content);
+				}

 				frappe.utils.copy_to_clipboard(url);

--- a/frappe/public/js/frappe/views/kanban/kanban_board.bundle.js
+++ b/frappe/public/js/frappe/views/kanban/kanban_board.bundle.js
@ -377,8 +377,13 @@ frappe.provide("frappe.views");
 		}

 		function bind_add_column() {
-			if (!self.board_perms.write) {
+			let doctype = self.cur_list.doctype;
+			let fieldname = self.cur_list.board.field_name;
+			const is_custom_field = frappe.meta.get_docfield(doctype, fieldname)?.is_custom_field;
+
+			if (!self.board_perms.write || !is_custom_field) {
 				// If no write access to board, editing board (by adding column) should be blocked
+				// If standard field then users can't add options
 				self.$kanban_board.find(".add-new-column").remove();
 				return;
 			}
--- a/frappe/public/js/frappe/views/workspace/workspace.js
+++ b/frappe/public/js/frappe/views/workspace/workspace.js
@ -188,6 +188,7 @@ frappe.views.Workspace = class Workspace {
 		$(".item-anchor").on("click", () => {
 			$(".list-sidebar.hidden-xs.hidden-sm").removeClass("opened");
 			$(".close-sidebar").css("display", "none");
+			$("body").css("overflow", "auto");
 		});

 		if (
--- a/frappe/public/js/frappe/widgets/quick_list_widget.js
+++ b/frappe/public/js/frappe/widgets/quick_list_widget.js
@ -133,6 +133,8 @@ export default class QuickListWidget extends Widget {
 	}

 	setup_quick_list_item(doc) {
+		const indicator = frappe.get_indicator(doc, this.document_type);
+
 		let $quick_list_item = $(`
 			<div class="quick-list-item">
 				<div class="ellipsis left">
@ -147,6 +149,14 @@ export default class QuickListWidget extends Widget {
 			</div>
 		`);

+		if (indicator) {
+			$(`
+				<div class="status indicator-pill ${indicator[1]} ellipsis">
+					${__(indicator[0])}
+				</div>
+			`).appendTo($quick_list_item);
+		}
+
 		$(`<div class="right-arrow">${frappe.utils.icon("right", "xs")}</div>`).appendTo(
 			$quick_list_item
 		);
--- a/frappe/public/js/frappe/widgets/shortcut_widget.js
+++ b/frappe/public/js/frappe/widgets/shortcut_widget.js
@ -79,7 +79,8 @@ export default class ShortcutWidget extends Widget {
 		});

 		let filters = frappe.utils.process_filter_expression(this.stats_filter);
-		if (this.type == "DocType" && filters) {
+
+		if (this.type == "DocType" && this.doc_view != "New" && filters) {
 			frappe.db
 				.count(this.link_to, {
 					filters: filters,
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@ -32,12 +32,45 @@ class WidgetDialog {
 	}

 	get_title() {
-		// DO NOT REMOVE: Comment to load translation
-		// __("New Chart") __("New Shortcut") __("Edit Chart") __("Edit Shortcut")
+		if (this.editing) {
+			switch (this.type) {
+				case "chart":
+					return __("Edit Chart");
+				case "shortcut":
+					return __("Edit Shortcut");
+				case "links":
+					return __("Edit Links");
+				case "number_card":
+					return __("Edit Number Card");
+				case "onboarding":
+					return __("Edit Onboarding");
+				case "quick_list":
+					return __("Edit Quick List");
+				case "custom_block":
+					return __("Edit Custom Block");
+				default:
+					return __("Edit {0}", [__(frappe.model.unscrub(this.type))]);
+			}
+		}

-		let action = this.editing ? "Edit" : "Add";
-		let label = (action = action + " " + frappe.model.unscrub(this.type));
-		return __(label);
+		switch (this.type) {
+			case "chart":
+				return __("New Chart");
+			case "shortcut":
+				return __("New Shortcut");
+			case "links":
+				return __("New Links");
+			case "number_card":
+				return __("New Number Card");
+			case "onboarding":
+				return __("New Onboarding");
+			case "quick_list":
+				return __("New Quick List");
+			case "custom_block":
+				return __("New Custom Block");
+			default:
+				return __("New {0}", [__(frappe.model.unscrub(this.type))]);
+		}
 	}

 	get_fields() {
--- a/frappe/public/js/integrations/google_drive_picker.js
+++ b/frappe/public/js/integrations/google_drive_picker.js
@ -42,11 +42,14 @@ export default class GoogleDrivePicker {
 	}

 	createPicker(access_token) {
-		this.view = new google.picker.View(google.picker.ViewId.DOCS);
+		const docsView = new google.picker.DocsView();
+		docsView.setParent("root"); // show the root folder by default
+		docsView.setIncludeFolders(true); // also show folders, not just files
+
 		this.picker = new google.picker.PickerBuilder()
 			.setAppId(this.appId)
 			.setOAuthToken(access_token)
-			.addView(this.view)
+			.addView(docsView)
 			.addView(new google.picker.DocsUploadView())
 			.setLocale(frappe.boot.lang)
 			.setCallback(this.pickerCallback)
--- a/frappe/public/js/onboarding_tours/onboarding_tours.js
+++ b/frappe/public/js/onboarding_tours/onboarding_tours.js
@ -265,7 +265,7 @@ frappe.ui.init_onboarding_tour = () => {
 	typeof frappe.boot.user.onboarding_status == "undefined" &&
 		frappe.boot.user.onboarding_status == {};
 	let route = frappe.router.current_route;
-	if (route[0] === "") return;
+	if (route?.[0] === "") return;

 	let tour_name;
 	let matching_tours = [];
--- a/frappe/public/scss/desk/data_import.scss
+++ b/frappe/public/scss/desk/data_import.scss
@ -14,4 +14,16 @@

 .table-preview {
 	margin-top: 12px;
+
+	.datatable .dt-scrollable .dt-row:last-child .dt-cell {
+		border-bottom: 1px solid var(--border-color);
+	}
+
+	.dt-row:last-child:not(.dt-row-filter) {
+		border-bottom: none;
+	}
+
+	.datatable .dt-header .dt-row-header {
+		background-color: unset;
+	}
 }
--- a/frappe/rate_limiter.py
+++ b/frappe/rate_limiter.py
@ -138,7 +138,7 @@ def rate_limit(
 			if not identity:
 				frappe.throw(_("Either key or IP flag is required."))

-			cache_key = f"rl:{frappe.form_dict.cmd}:{identity}"
+			cache_key = frappe.cache.make_key(f"rl:{frappe.form_dict.cmd}:{identity}")

 			value = frappe.cache.get(cache_key)
 			if not value:
--- a/frappe/recorder.py
+++ b/frappe/recorder.py
@ -1,32 +1,67 @@
 # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import datetime
+import cProfile
 import functools
 import inspect
+import io
 import json
+import pstats
 import re
 import time
 from collections import Counter
 from collections.abc import Callable
+from dataclasses import dataclass

 import sqlparse

 import frappe
 from frappe import _
 from frappe.database.database import is_query_type
+from frappe.utils import now_datetime

 RECORDER_INTERCEPT_FLAG = "recorder-intercept"
+RECORDER_CONFIG_FLAG = "recorder-config"
 RECORDER_REQUEST_SPARSE_HASH = "recorder-requests-sparse"
 RECORDER_REQUEST_HASH = "recorder-requests"
 TRACEBACK_PATH_PATTERN = re.compile(".*/apps/")
+RECORDER_AUTO_DISABLE = 5 * 60


-def sql(*args, **kwargs):
+@dataclass
+class RecorderConfig:
+	record_requests: bool = True  # Record web request
+	record_jobs: bool = True  # record background jobs
+	record_sql: bool = True  # Record SQL queries
+	capture_stack: bool = True  # Recod call stack of SQL queries
+	profile: bool = False  # Run cProfile
+	explain: bool = True  # Provide explain output of SQL queries
+	request_filter: str = "/"  # Filter request paths
+	jobs_filter: str = ""  # Filter background jobs
+
+	def __post_init__(self):
+		if not (self.record_jobs or self.record_requests):
+			frappe.throw("You must record one of jobs or requests")
+
+	def store(self):
+		frappe.cache.set_value(RECORDER_CONFIG_FLAG, self, expires_in_sec=RECORDER_AUTO_DISABLE)
+
+	@classmethod
+	def retrieve(cls):
+		return frappe.cache.get_value(RECORDER_CONFIG_FLAG) or cls()
+
+	@staticmethod
+	def delete():
+		frappe.cache.delete_value(RECORDER_CONFIG_FLAG)
+
+
+def record_sql(*args, **kwargs):
 	start_time = time.monotonic()
 	result = frappe.db._sql(*args, **kwargs)
 	end_time = time.monotonic()

-	stack = list(get_current_stack_frames())
+	stack = []
+	if frappe.local._recorder.config.capture_stack:
+		stack = list(get_current_stack_frames())

 	data = {
 		"query": str(frappe.db.last_query),
@ -69,6 +104,7 @@ def post_process():
 	frappe.db.rollback()
 	frappe.db.begin(read_only=True)  # Explicitly start read only transaction

+	config = RecorderConfig.retrieve()
 	result = list(frappe.cache.hgetall(RECORDER_REQUEST_HASH).values())

 	for request in result:
@ -79,7 +115,7 @@ def post_process():
 			call["query"] = formatted_query

 			# Collect EXPLAIN for executed query
-			if is_query_type(formatted_query, ("select", "update", "delete")):
+			if config.explain and is_query_type(formatted_query, ("select", "update", "delete")):
 				# Only SELECT/UPDATE/DELETE queries can be "EXPLAIN"ed
 				try:
 					call["explain_result"] = frappe.db.sql(f"EXPLAIN {formatted_query}", as_dict=True)
@ -88,6 +124,8 @@ def post_process():
 		mark_duplicates(request)
 		frappe.cache.hset(RECORDER_REQUEST_HASH, request["uuid"], request)

+	config.delete()
+

 def mark_duplicates(request):
 	exact_duplicates = Counter([call["query"] for call in request["calls"]])
@ -134,7 +172,7 @@ def normalize_query(query: str) -> str:
 def record(force=False):
 	if __debug__:
 		if frappe.cache.get_value(RECORDER_INTERCEPT_FLAG) or force:
-			frappe.local._recorder = Recorder()
+			frappe.local._recorder = Recorder(force=force)


 def dump():
@ -144,38 +182,78 @@ def dump():


 class Recorder:
-	def __init__(self):
-		self.uuid = frappe.generate_hash(length=10)
-		self.time = datetime.datetime.now()
+	def __init__(self, force=False):
+		self.config = RecorderConfig.retrieve()
 		self.calls = []
-		if frappe.request:
+		self._patched_sql = False
+		self.profiler = None
+		self._recording = True
+		self.force = force
+		self.cmd = None
+		self.method = None
+		self.headers = None
+		self.form_dict = None
+
+		if (
+			self.config.record_requests
+			and frappe.request
+			and self.config.request_filter in frappe.request.path
+		):
 			self.path = frappe.request.path
 			self.cmd = frappe.local.form_dict.cmd or ""
 			self.method = frappe.request.method
 			self.headers = dict(frappe.local.request.headers)
 			self.form_dict = frappe.local.form_dict
 			self.event_type = "HTTP Request"
-		elif frappe.job:
+		elif self.config.record_jobs and frappe.job and self.config.jobs_filter in frappe.job.method:
 			self.event_type = "Background Job"
 			self.path = frappe.job.method
 			self.cmd = None
 			self.method = None
 			self.headers = None
 			self.form_dict = None
+		elif not self.force:
+			self._recording = False
+			return
 		else:
-			self.event_type = None
-			self.path = None
-			self.cmd = None
-			self.method = None
-			self.headers = None
-			self.form_dict = None
+			self.event_type = "Function Call"

-		_patch()
+		self.uuid = frappe.generate_hash(length=10)
+		self.time = now_datetime()
+
+		if self.config.record_sql:
+			self._patch_sql()
+			self._patched_sql = True
+
+		if self.config.profile:
+			self.profiler = cProfile.Profile()
+			self.profiler.enable()

 	def register(self, data):
 		self.calls.append(data)

+	def cleanup(self):
+		if self.profiler:
+			self.profiler.disable()
+		if self._patched_sql:
+			self._unpatch_sql()
+
+	def process_profiler(self):
+		if self.config.profile or self.profiler:
+			self.profiler.disable()
+			profiler_output = io.StringIO()
+			pstats.Stats(self.profiler, stream=profiler_output).strip_dirs().sort_stats(
+				"cumulative"
+			).print_stats()
+			profile = profiler_output.getvalue()
+			profiler_output.close()
+			return profile
+
 	def dump(self):
+		if not self._recording:
+			return
+		profiler_output = self.process_profiler()
+
 		request_data = {
 			"uuid": self.uuid,
 			"path": self.path,
@ -183,38 +261,37 @@ class Recorder:
 			"time": self.time,
 			"queries": len(self.calls),
 			"time_queries": float("{:0.3f}".format(sum(call["duration"] for call in self.calls))),
-			"duration": float(f"{(datetime.datetime.now() - self.time).total_seconds() * 1000:0.3f}"),
+			"duration": float(f"{(now_datetime() - self.time).total_seconds() * 1000:0.3f}"),
 			"method": self.method,
 			"event_type": self.event_type,
 		}
 		frappe.cache.hset(RECORDER_REQUEST_SPARSE_HASH, self.uuid, request_data)
-		frappe.publish_realtime(
-			event="recorder-dump-event",
-			message=json.dumps(request_data, default=str),
-			user="Administrator",
-		)

 		request_data["calls"] = self.calls
 		request_data["headers"] = self.headers
 		request_data["form_dict"] = self.form_dict
+		request_data["profile"] = profiler_output
 		frappe.cache.hset(RECORDER_REQUEST_HASH, self.uuid, request_data)

+		if self.config.record_sql:
+			self._unpatch_sql()

-def _patch():
-	frappe.db._sql = frappe.db.sql
-	frappe.db.sql = sql
+	@staticmethod
+	def _patch_sql():
+		frappe.db._sql = frappe.db.sql
+		frappe.db.sql = record_sql

-
-def _unpatch():
-	frappe.db.sql = frappe.db._sql
+	@staticmethod
+	def _unpatch_sql():
+		frappe.db.sql = frappe.db._sql


 def do_not_record(function):
 	@functools.wraps(function)
 	def wrapper(*args, **kwargs):
 		if hasattr(frappe.local, "_recorder"):
+			frappe.local._recorder.cleanup()
 			del frappe.local._recorder
-			frappe.db.sql = frappe.db._sql
 		return function(*args, **kwargs)

 	return wrapper
@ -240,8 +317,29 @@ def status(*args, **kwargs):
@frappe.whitelist()
@do_not_record
@administrator_only
-def start(*args, **kwargs):
-	frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1, expires_in_sec=60 * 60)
+def start(
+	record_jobs: bool = True,
+	record_requests: bool = True,
+	record_sql: bool = True,
+	profile: bool = False,
+	capture_stack: bool = True,
+	explain: bool = True,
+	request_filter: str = "/",
+	jobs_filter: str = "",
+	*args,
+	**kwargs,
+):
+	RecorderConfig(
+		record_requests=int(record_requests),
+		record_jobs=int(record_jobs),
+		record_sql=int(record_sql),
+		profile=int(profile),
+		capture_stack=int(capture_stack),
+		explain=int(explain),
+		request_filter=request_filter,
+		jobs_filter=jobs_filter,
+	).store()
+	frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1, expires_in_sec=RECORDER_AUTO_DISABLE)


@frappe.whitelist()
@ -287,7 +385,7 @@ def record_queries(func: Callable):
 		frappe.local._recorder.path = f"Function call: {func.__module__}.{func.__qualname__}"
 		ret = func(*args, **kwargs)
 		dump()
-		_unpatch()
+		Recorder._unpatch_sql()
 		post_process()
 		print("Recorded queries, open recorder to view them.")
 		return ret
--- a/frappe/templates/includes/login/login.js
+++ b/frappe/templates/includes/login/login.js
@ -21,7 +21,8 @@ login.bind_events = function () {
 		args.usr = frappe.utils.xss_sanitise(($("#login_email").val() || "").trim());
 		args.pwd = $("#login_password").val();
 		if (!args.usr || !args.pwd) {
-			frappe.msgprint('{{ _("Both login and password required") }}');
+			{# striptags is used to remove newlines, e is used for escaping #}
+			frappe.msgprint("{{ _('Both login and password required') | striptags | e }}");
 			return false;
 		}
 		login.call(args, null, "/login");
@ -36,7 +37,7 @@ login.bind_events = function () {
 		args.redirect_to = frappe.utils.sanitise_redirect(frappe.utils.get_url_arg("redirect-to"));
 		args.full_name = frappe.utils.xss_sanitise(($("#signup_fullname").val() || "").trim());
 		if (!args.email || !validate_email(args.email) || !args.full_name) {
-			login.set_status('{{ _("Valid email and name required") }}', 'red');
+			login.set_status({{ _("Valid email and name required") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@ -49,7 +50,7 @@ login.bind_events = function () {
 		args.cmd = "frappe.core.doctype.user.user.reset_password";
 		args.user = ($("#forgot_email").val() || "").trim();
 		if (!args.user) {
-			login.set_status('{{ _("Valid Login id required.") }}', 'red');
+			login.set_status({{ _("Valid Login id required.") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@ -62,14 +63,14 @@ login.bind_events = function () {
 		args.cmd = "frappe.www.login.send_login_link";
 		args.email = ($("#login_with_email_link_email").val() || "").trim();
 		if (!args.email) {
-			login.set_status('{{ _("Valid Login id required.") }}', 'red');
+			login.set_status({{ _("Valid Login id required.") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args).then(() => {
-			login.set_status('{{ _("Login link sent to your email") }}', 'blue');
+			login.set_status({{ _("Login link sent to your email") | tojson }}, 'blue');
 			$("#login_with_email_link_email").val("");
 		}).catch(() => {
-			login.set_status('{{ _("Send login link") }}', 'blue');
+			login.set_status({{ _("Send login link") | tojson }}, 'blue');
 		});

 		return false;
@ -79,10 +80,10 @@ login.bind_events = function () {
 		var input = $($(this).attr("toggle"));
 		if (input.attr("type") == "password") {
 			input.attr("type", "text");
-			$(this).text('{{ _("Hide") }}')
+			$(this).text({{ _("Hide") | tojson }})
 		} else {
 			input.attr("type", "password");
-			$(this).text('{{ _("Show") }}')
+			$(this).text({{ _("Show") | tojson }})
 		}
 	});

@ -93,7 +94,7 @@ login.bind_events = function () {
 		args.usr = ($("#login_email").val() || "").trim();
 		args.pwd = $("#login_password").val();
 		if (!args.usr || !args.pwd) {
-			login.set_status('{{ _("Both login and password required") }}', 'red');
+			login.set_status({{ _("Both login and password required") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@ -168,7 +169,7 @@ login.signup = function () {

 // Login
 login.call = function (args, callback, url="/") {
-	login.set_status('{{ _("Verifying...") }}', 'blue');
+	login.set_status({{ _("Verifying...") | tojson }}, 'blue');

 	return frappe.call({
 		type: "POST",
@ -227,13 +228,13 @@ login.login_handlers = (function () {
 	var login_handlers = {
 		200: function (data) {
 			if (data.message == 'Logged In') {
-				login.set_status('{{ _("Success") }}', 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 				document.body.innerHTML = `{% include "templates/includes/splash_screen.html" %}`;
 				window.location.href = frappe.utils.sanitise_redirect(frappe.utils.get_url_arg("redirect-to")) || data.home_page;
 			} else if (data.message == 'Password Reset') {
 				window.location.href = frappe.utils.sanitise_redirect(data.redirect_to);
 			} else if (data.message == "No App") {
-				login.set_status("{{ _('Success') }}", 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 				if (localStorage) {
 					var last_visited =
 						localStorage.getItem("last_visited")
@ -252,13 +253,13 @@ login.login_handlers = (function () {
 				}
 			} else if (window.location.hash === '#forgot') {
 				if (data.message === 'not found') {
-					login.set_status('{{ _("Not a valid user") }}', 'red');
+					login.set_status({{ _("Not a valid user") | tojson }}, 'red');
 				} else if (data.message == 'not allowed') {
-					login.set_status('{{ _("Not Allowed") }}', 'red');
+					login.set_status({{ _("Not Allowed") | tojson }}, 'red');
 				} else if (data.message == 'disabled') {
-					login.set_status('{{ _("Not Allowed: Disabled User") }}', 'red');
+					login.set_status({{ _("Not Allowed: Disabled User") | tojson }}, 'red');
 				} else {
-					login.set_status('{{ _("Instructions Emailed") }}', 'green');
+					login.set_status({{ _("Instructions Emailed") | tojson }}, 'green');
 				}


@ -266,7 +267,7 @@ login.login_handlers = (function () {
 				if (cint(data.message[0]) == 0) {
 					login.set_status(data.message[1], 'red');
 				} else {
-					login.set_status('{{ _("Success") }}', 'green');
+					login.set_status({{ _("Success") | tojson }}, 'green');
 					frappe.msgprint(data.message[1])
 				}
 				//login.set_status(__(data.message), 'green');
@ -274,7 +275,7 @@ login.login_handlers = (function () {

 			//OTP verification
 			if (data.verification && data.message != 'Logged In') {
-				login.set_status('{{ _("Success") }}', 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');

 				document.cookie = "tmp_id=" + data.tmp_id;

@ -287,10 +288,10 @@ login.login_handlers = (function () {
 				}
 			}
 		},
-		401: get_error_handler('{{ _("Invalid Login. Try again.") }}'),
-		417: get_error_handler('{{ _("Oops! Something went wrong.") }}'),
-		404: get_error_handler('{{ _("User does not exist.")}}'),
-		500: get_error_handler('{{ _("Something went wrong.") }}')
+		401: get_error_handler({{ _("Invalid Login. Try again.") | tojson }}),
+		417: get_error_handler({{ _("Oops! Something went wrong.") | tojson }}),
+		404: get_error_handler({{ _("User does not exist.") | tojson }}),
+		500: get_error_handler({{ _("Something went wrong.") | tojson }})
 	};

 	return login_handlers;
@ -322,7 +323,8 @@ var verify_token = function (event) {
 		args.otp = $("#login_token").val();
 		args.tmp_id = frappe.get_cookie('tmp_id');
 		if (!args.otp) {
-			frappe.msgprint('{{ _("Login token required") }}');
+			{# striptags is used to remove newlines, e is used for escaping #}
+			frappe.msgprint("{{ _('Login token required') | striptags | e }}");
 			return false;
 		}
 		login.call(args);
@ -336,11 +338,11 @@ var request_otp = function (r) {
 		`<div id="twofactor_div">
 			<form class="form-verify">
 				<div class="page-card-head">
-					<span class="indicator blue" data-text="Verification">{{ _("Verification") }}</span>
+					<span class="indicator blue" data-text="Verification">{{ _("Verification") | e }}</span>
 				</div>
 				<div id="otp_div"></div>
-				<input type="text" id="login_token" autocomplete="off" class="form-control" placeholder="{{ _("Verification Code") }}" required="">
-				<button class="btn btn-sm btn-primary btn-block mt-3" id="verify_token">{{ _("Verify") }}</button>
+				<input type="text" id="login_token" autocomplete="off" class="form-control" placeholder="{{ _("Verification Code") | e }}" required="">
+				<button class="btn btn-sm btn-primary btn-block mt-3" id="verify_token">{{ _("Verify") | e }}</button>
 			</form>
 		</div>`
 	);
@ -354,11 +356,11 @@ var continue_otp_app = function (setup, qrcode) {
 	var qrcode_div = $('<div class="text-muted" style="padding-bottom: 15px;"></div>');

 	if (setup) {
-		direction = $('<div>').attr('id', 'qr_info').html('{{ _("Enter Code displayed in OTP App.") }}');
+		direction = $('<div>').attr('id', 'qr_info').text({{ _("Enter Code displayed in OTP App.") | tojson }});
 		qrcode_div.append(direction);
 		$('#otp_div').prepend(qrcode_div);
 	} else {
-		direction = $('<div>').attr('id', 'qr_info').html('{{ _("OTP setup using OTP App was not completed. Please contact Administrator.") }}');
+		direction = $('<div>').attr('id', 'qr_info').text({{ _("OTP setup using OTP App was not completed. Please contact Administrator.") | tojson }});
 		qrcode_div.append(direction);
 		$('#otp_div').prepend(qrcode_div);
 	}
@ -372,7 +374,7 @@ var continue_sms = function (setup, prompt) {
 		sms_div.append(prompt)
 		$('#otp_div').prepend(sms_div);
 	} else {
-		direction = $('<div>').attr('id', 'qr_info').html(prompt || '{{ _("SMS was not sent. Please contact Administrator.") }}');
+		direction = $('<div>').attr('id', 'qr_info').html(prompt || {{ _("SMS was not sent. Please contact Administrator.") | tojson }});
 		sms_div.append(direction);
 		$('#otp_div').prepend(sms_div)
 	}
@ -386,7 +388,7 @@ var continue_email = function (setup, prompt) {
 		email_div.append(prompt)
 		$('#otp_div').prepend(email_div);
 	} else {
-		var direction = $('<div>').attr('id', 'qr_info').html(prompt || '{{ _("Verification code email not sent. Please contact Administrator.") }}');
+		var direction = $('<div>').attr('id', 'qr_info').html(prompt || {{ _("Verification code email not sent. Please contact Administrator.") | tojson }});
 		email_div.append(direction);
 		$('#otp_div').prepend(email_div);
 	}
--- a/frappe/test_runner.py
+++ b/frappe/test_runner.py
@ -38,6 +38,7 @@ def xmlrunner_wrapper(output):


 def main(
+	site=None,
 	app=None,
 	module=None,
 	doctype=None,
@ -50,9 +51,18 @@ def main(
 	doctype_list_path=None,
 	failfast=False,
 	case=None,
+	skip_test_records=False,
+	skip_before_tests=False,
 ):
 	global unittest_runner

+	frappe.init(site=site)
+	if not frappe.db:
+		frappe.connect()
+
+	frappe.flags.skip_before_tests = skip_before_tests
+	frappe.flags.skip_test_records = skip_test_records
+
 	if doctype_list_path:
 		app, doctype_list_path = doctype_list_path.split(os.path.sep, 1)
 		with open(frappe.get_app_path(app, doctype_list_path)) as f:
@ -69,9 +79,6 @@ def main(
 		frappe.flags.print_messages = verbose
 		frappe.flags.in_test = True

-		if not frappe.db:
-			frappe.connect()
-
 		# workaround! since there is no separate test db
 		frappe.clear_cache()
 		scheduler_disabled_by_user = frappe.utils.scheduler.is_scheduler_disabled(verbose=False)
@ -89,9 +96,22 @@ def main(
 				doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output
 			)
 		elif module_def:
-			doctypes = frappe.db.get_list(
-				"DocType", filters={"module": module_def, "istable": 0}, pluck="name"
+			doctypes = []
+			doctypes_ = frappe.get_list(
+				"DocType",
+				filters={"module": module_def, "istable": 0},
+				fields=["name", "module"],
+				as_list=True,
 			)
+			for doctype, module in doctypes_:
+				test_module = get_module_name(doctype, module, "test_", app=app)
+				try:
+					importlib.import_module(test_module)
+				except Exception:
+					pass
+				else:
+					doctypes.append(doctype)
+
 			ret = run_tests_for_doctype(
 				doctypes, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output
 			)
@ -329,9 +349,6 @@ def _add_test(app, path, filename, verbose, test_suite=None):


 def make_test_records(doctype, verbose=0, force=False, commit=False):
-	if not frappe.db:
-		frappe.connect()
-
 	if frappe.flags.skip_test_records:
 		return

--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@ -15,7 +15,7 @@ from werkzeug.test import TestResponse
 import frappe
 from frappe.installer import update_site_config
 from frappe.tests.utils import FrappeTestCase, patch_hooks
-from frappe.utils import cint, get_test_client, get_url
+from frappe.utils import cint, get_site_url, get_test_client, get_url

 try:
 	_site = frappe.local.site
@ -432,6 +432,21 @@ class TestResponse(FrappeAPITestCase):
 		self.assertGreater(cint(response.headers["content-length"]), 0)
 		self.assertEqual(response.headers["content-disposition"], f'filename="{encoded_filename}"')

+	def test_download_private_file_with_unique_url(self):
+		test_content = frappe.generate_hash()
+		file = frappe.get_doc(
+			{
+				"doctype": "File",
+				"file_name": test_content,
+				"content": test_content,
+				"is_private": 1,
+			}
+		)
+		file.insert()
+
+		self.assertEqual(self.get(file.unique_url, {"sid": self.sid}).text, test_content)
+		self.assertEqual(self.get(file.file_url, {"sid": self.sid}).text, test_content)
+

 def generate_admin_keys():
 	from frappe.core.doctype.user.user import generate_keys
--- a/frappe/tests/test_commands.py
+++ b/frappe/tests/test_commands.py
@ -173,14 +173,18 @@ class BaseTestCommands(FrappeTestCase):
 		cmd_config = {
 			"test_site": TEST_SITE,
 			"admin_password": frappe.conf.admin_password,
-			"root_login": frappe.conf.root_login,
-			"root_password": frappe.conf.root_password,
 			"db_type": frappe.conf.db_type,
+			"db_root_username": frappe.conf.root_login,
+			"db_root_password": frappe.conf.root_password,
 		}

 		if not os.path.exists(os.path.join(TEST_SITE, "site_config.json")):
 			cls.execute(
-				"bench new-site {test_site} --admin-password {admin_password} --db-type" " {db_type}",
+				"bench new-site {test_site} "
+				"--admin-password {admin_password} "
+				"--db-root-username {db_root_username} "
+				"--db-root-password {db_root_password} "
+				"--db-type {db_type}",
 				cmd_config,
 			)

--- a/frappe/tests/test_db_query.py
+++ b/frappe/tests/test_db_query.py
@ -1255,6 +1255,9 @@ class TestReportView(FrappeTestCase):
 			response = execute_cmd("frappe.desk.reportview.get")
 			self.assertNotIn("published", response["keys"])

+			# If none of the fields are accessible then result should be empty
+			self.assertEqual(frappe.get_list("Blog Post", "published"), [])
+
 	def test_reportview_get_admin(self):
 		# Admin should be able to see access all fields
 		with setup_patched_blog_post():
--- a/frappe/tests/test_fmt_money.py
+++ b/frappe/tests/test_fmt_money.py
@ -100,10 +100,3 @@ class TestFmtMoney(FrappeTestCase):
 		frappe.db.set_value("Currency", "JPY", "symbol_on_right", 1)
 		self.assertEqual(fmt_money(100.0, format="#,###.##", currency="JPY"), "100.00 ¥")
 		self.assertEqual(fmt_money(100.0, format="#,###.##", currency="USD"), "$ 100.00")
-
-
-if __name__ == "__main__":
-	import unittest
-
-	frappe.connect()
-	unittest.main()
--- a/frappe/tests/test_model_utils.py
+++ b/frappe/tests/test_model_utils.py
@ -36,10 +36,10 @@ class TestModelUtils(FrappeTestCase):
 		todo_all_columns = frappe.get_meta("ToDo").get_valid_columns()
 		self.assertListEqual(todo_all_fields, todo_all_columns)

-		# Guest should have access to no fields in ToDo
+		# Guest should have access to no non-std fields in ToDo
 		with set_user("Guest"):
 			guest_permitted_fields = get_permitted_fields("ToDo")
-			self.assertEqual(guest_permitted_fields, [])
+			self.assertNotIn("description", guest_permitted_fields)

 		# everyone should have access to all fields of core doctypes
 		with set_user("Guest"):
@ -55,15 +55,14 @@ class TestModelUtils(FrappeTestCase):
 				"Installed Application", parenttype="Installed Applications"
 			)
 			child_all_fields = frappe.get_meta("Installed Application").get_valid_columns()
-			self.assertEqual(without_parent_fields, [])
 			self.assertLess(len(without_parent_fields), len(with_parent_fields))
 			self.assertSequenceEqual(set(with_parent_fields), set(child_all_fields))

-		# guest has access to no fields
+		# guest has access to no non-std fields
 		with set_user("Guest"):
-			self.assertEqual(get_permitted_fields("Installed Application"), [])
-			self.assertEqual(
-				get_permitted_fields("Installed Application", parenttype="Installed Applications"), []
+			self.assertNotIn("app_name", get_permitted_fields("Installed Application"))
+			self.assertNotIn(
+				"app_name", get_permitted_fields("Installed Application", parenttype="Installed Applications")
 			)

 	def test_is_default_field(self):
--- a/frappe/translate.py
+++ b/frappe/translate.py
@ -20,35 +20,11 @@ from csv import reader, writer

 import frappe
 from frappe.gettext.extractors.javascript import extract_javascript
+from frappe.gettext.extractors.utils import extract_messages_from_code, is_translatable
 from frappe.gettext.translate import get_translations_from_mo
-from frappe.model.utils import InvalidIncludePath, render_include
 from frappe.query_builder import DocType, Field
 from frappe.utils import cstr, get_bench_path, is_html, strip, strip_html_tags, unique

-TRANSLATE_PATTERN = re.compile(
-	r"_\(\s*"  # starts with literal `_(`, ignore following whitespace/newlines
-	# BEGIN: message search
-	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
-	r"(?P<message>((?!\1).)*)"  # Keep matching until string closing identifier is met which is same as 1st capture group
-	r"\1"  # match exact string closing identifier
-	# END: message search
-	# BEGIN: python context search
-	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
-	r"([\"'])"  # start of context string identifier; 5th capture group
-	r"(?P<py_context>((?!\5).)*)"  # capture context string till closing id is found
-	r"\5"  # match context string closure
-	r")?"  # match 0 or 1 context strings
-	# END: python context search
-	# BEGIN: JS context search
-	r"(\s*,\s*(.)*?\s*(,\s*"  # skip message format replacements: ["format", ...] | null | []
-	r"([\"'])"  # start of context string; 11th capture group
-	r"(?P<js_context>((?!\11).)*)"  # capture context string till closing id is found
-	r"\11"  # match context string closure
-	r")*"
-	r")*"  # match one or more context string
-	# END: JS context search
-	r"\s*\)"  # Closing function call ignore leading whitespace/newlines
-)
 REPORT_TRANSLATE_PATTERN = re.compile('"([^:,^"]*):')
 CSV_STRIP_WHITESPACE_PATTERN = re.compile(r"{\s?([0-9]+)\s?}")

@ -239,9 +215,6 @@ def get_translation_dict_from_file(path, lang, app, throw=False) -> dict[str, st


 def get_user_translations(lang):
-	if not frappe.db:
-		frappe.connect()
-
 	def _read_from_db():
 		user_translations = {}
 		translations = frappe.get_all(
@ -676,59 +649,6 @@ def extract_messages_from_javascript_code(code: str) -> list[tuple[int, str, str
 	return messages


-def extract_messages_from_code(code):
-	"""
-	Extracts translatable strings from a code file
-	:param code: code from which translatable files are to be extracted
-	"""
-	from jinja2 import TemplateError
-
-	try:
-		code = frappe.as_unicode(render_include(code))
-
-	# Exception will occur when it encounters John Resig's microtemplating code
-	except (TemplateError, ImportError, InvalidIncludePath, OSError) as e:
-		if isinstance(e, InvalidIncludePath):
-			frappe.clear_last_message()
-
-	messages = []
-
-	for m in TRANSLATE_PATTERN.finditer(code):
-		message = m.group("message")
-		context = m.group("py_context") or m.group("js_context")
-		pos = m.start()
-
-		if is_translatable(message):
-			messages.append([pos, message, context])
-
-	return add_line_number(messages, code)
-
-
-def is_translatable(m):
-	if (
-		re.search("[a-zA-Z]", m)
-		and not m.startswith("fa fa-")
-		and not m.endswith("px")
-		and not m.startswith("eval:")
-	):
-		return True
-	return False
-
-
-def add_line_number(messages, code):
-	ret = []
-	messages = sorted(messages, key=lambda x: x[0])
-	newlines = [m.start() for m in re.compile(r"\n").finditer(code)]
-	line = 1
-	newline_i = 0
-	for pos, message, context in messages:
-		while newline_i < len(newlines) and pos > newlines[newline_i]:
-			line += 1
-			newline_i += 1
-		ret.append([line, message, context])
-	return ret
-
-
 def read_csv_file(path):
 	"""Read CSV file and return as list of list

@ -1054,9 +974,6 @@ def get_all_languages(with_language_name: bool = False) -> list:
 	def get_all_language_with_name():
 		return frappe.get_all("Language", ["language_code", "language_name"], {"enabled": 1})

-	if not frappe.db:
-		frappe.connect()
-
 	if with_language_name:
 		return frappe.cache.get_value("languages_with_name", get_all_language_with_name)
 	else:
@ -1106,44 +1023,6 @@ def print_language(language: str):
 	frappe.local.jenv = _jenv


-@functools.total_ordering
-class LazyTranslate:
-	__slots__ = ("msg", "lang", "context")
-
-	def __init__(self, msg: str, lang: str | None = None, context: str | None = None) -> None:
-		self.msg = msg
-		self.lang = lang
-		self.context = context
-
-	@property
-	def value(self) -> str:
-		return frappe._(str(self.msg), self.lang, self.context)
-
-	def __str__(self):
-		return self.value
-
-	def __add__(self, other):
-		if isinstance(other, (str, LazyTranslate)):
-			return self.value + str(other)
-		raise NotImplementedError
-
-	def __radd__(self, other):
-		if isinstance(other, (str, LazyTranslate)):
-			return str(other) + self.value
-		return NotImplementedError
-
-	def __repr__(self) -> str:
-		return f"'{self.value}'"
-
-	# NOTE: it's required to override these methods and raise error as default behaviour will
-	# return `False` in all cases.
-	def __eq__(self, other):
-		raise NotImplementedError
-
-	def __lt__(self, other):
-		raise NotImplementedError
-
-
 # Backward compatibility
 get_full_dict = get_all_translations
 load_lang = get_translations_from_apps
--- a/frappe/twofactor.py
+++ b/frappe/twofactor.py
@ -43,11 +43,9 @@ def toggle_two_factor_auth(state, roles=None):

 def two_factor_is_enabled(user=None):
 	"""Return True if 2FA is enabled."""
-	enabled = cint(frappe.db.get_single_value("System Settings", "enable_two_factor_auth"))
+	enabled = cint(frappe.get_system_settings("enable_two_factor_auth"))
 	if enabled:
-		bypass_two_factor_auth = cint(
-			frappe.db.get_single_value("System Settings", "bypass_2fa_for_retricted_ip_users")
-		)
+		bypass_two_factor_auth = cint(frappe.get_system_settings("bypass_2fa_for_retricted_ip_users"))
 		if bypass_two_factor_auth and user:
 			user_doc = frappe.get_doc("User", user)
 			restrict_ip_list = (
@ -144,7 +142,7 @@ def get_otpsecret_for_(user):


 def get_verification_method():
-	return frappe.db.get_single_value("System Settings", "two_factor_method")
+	return frappe.get_system_settings("two_factor_method")


 def confirm_otp_token(login_manager, otp=None, tmp_id=None):
@ -190,7 +188,7 @@ def confirm_otp_token(login_manager, otp=None, tmp_id=None):


 def get_verification_obj(user, token, otp_secret):
-	otp_issuer = frappe.db.get_single_value("System Settings", "otp_issuer_name")
+	otp_issuer = frappe.get_system_settings("otp_issuer_name")
 	verification_method = get_verification_method()
 	verification_obj = None
 	if verification_method == "SMS":
@ -263,7 +261,7 @@ def process_2fa_for_email(user, token, otp_secret, otp_issuer, method="Email"):
 def get_email_subject_for_2fa(kwargs_dict):
 	"""Get email subject for 2fa."""
 	subject_template = _("Login Verification Code from {}").format(
-		frappe.db.get_single_value("System Settings", "otp_issuer_name")
+		frappe.get_system_settings("otp_issuer_name")
 	)
 	return frappe.render_template(subject_template, kwargs_dict)

@ -281,7 +279,7 @@ def get_email_body_for_2fa(kwargs_dict):
 def get_email_subject_for_qr_code(kwargs_dict):
 	"""Get QRCode email subject."""
 	subject_template = _("One Time Password (OTP) Registration Code from {}").format(
-		frappe.db.get_single_value("System Settings", "otp_issuer_name")
+		frappe.get_system_settings("otp_issuer_name")
 	)
 	return frappe.render_template(subject_template, kwargs_dict)

@ -299,7 +297,7 @@ def get_link_for_qrcode(user, totp_uri):
 	key = frappe.generate_hash(length=20)
 	key_user = f"{key}_user"
 	key_uri = f"{key}_uri"
-	lifespan = int(frappe.db.get_single_value("System Settings", "lifespan_qrcode_image")) or 240
+	lifespan = int(frappe.get_system_settings("lifespan_qrcode_image")) or 240
 	frappe.cache.set_value(key_uri, totp_uri, expires_in_sec=lifespan)
 	frappe.cache.set_value(key_user, user, expires_in_sec=lifespan)
 	return get_url(f"/qrcode?k={key}")
@ -433,7 +431,7 @@ def should_remove_barcode_image(barcode):
 	"""Check if it's time to delete barcode image from server."""
 	if isinstance(barcode, str):
 		barcode = frappe.get_doc("File", barcode)
-	lifespan = frappe.db.get_single_value("System Settings", "lifespan_qrcode_image") or 240
+	lifespan = frappe.get_system_settings("lifespan_qrcode_image") or 240
 	if time_diff_in_seconds(get_datetime(), barcode.creation) > int(lifespan):
 		return True
 	return False
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@ -370,6 +370,8 @@ class BackupGenerator:
 			n.write(c.read())

 	def take_dump(self):
+		import shlex
+
 		import frappe.utils
 		from frappe.utils.change_log import get_app_branch

@ -419,15 +421,15 @@ class BackupGenerator:
 		extra = []
 		if self.db_type == "mariadb":
 			if self.backup_includes:
-				extra.extend([f"'{x}'" for x in self.backup_includes])
+				extra.extend(self.backup_includes)
 			elif self.backup_excludes:
-				extra.extend([f"--ignore-table='{self.db_name}.{table}'" for table in self.backup_excludes])
+				extra.extend([f"--ignore-table={self.db_name}.{table}" for table in self.backup_excludes])

 		elif self.db_type == "postgres":
 			if self.backup_includes:
-				extra.extend([f"--table='public.\"{table}\"'" for table in self.backup_includes])
+				extra.extend([f'--table=public."{table}"' for table in self.backup_includes])
 			elif self.backup_excludes:
-				extra.extend([f"--exclude-table-data='public.\"{table}\"'" for table in self.backup_excludes])
+				extra.extend([f'--exclude-table-data=public."{table}"' for table in self.backup_excludes])

 		from frappe.database import get_command

@ -446,11 +448,11 @@ class BackupGenerator:
 				exc=frappe.ExecutableNotFound,
 			)
 		cmd.append(bin)
-		cmd.extend(args)
+		cmd.append(shlex.join(args))

 		command = " ".join(["set -o pipefail;"] + cmd + ["|", gzip_exc, ">>", self.backup_path_db])
 		if self.verbose:
-			print(command.replace(frappe.utils.esc(self.password, "$ "), "*" * 10) + "\n")
+			print(command.replace(shlex.quote(self.password), "*" * 10) + "\n")

 		frappe.utils.execute_in_shell(command, low_priority=True, check_exit_code=True)

--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@ -2486,18 +2486,27 @@ def is_site_link(link: str) -> bool:
 	return urlparse(link).netloc == urlparse(frappe.utils.get_url()).netloc


-def add_trackers_to_url(url: str, source: str, campaign: str, medium: str = "email") -> str:
+def add_trackers_to_url(
+	url: str,
+	source: str,
+	campaign: str | None = None,
+	medium: str | None = None,
+	content: str | None = None,
+) -> str:
 	url_parts = list(urlparse(url))
 	if url_parts[0] == "mailto":
 		return url

-	trackers = {
-		"source": source,
-		"medium": medium,
-	}
+	trackers = {"utm_source": source}
+
+	if medium:
+		trackers["utm_medium"] = medium

 	if campaign:
-		trackers["campaign"] = campaign
+		trackers["utm_campaign"] = campaign
+
+	if content:
+		trackers["utm_content"] = content

 	query = dict(parse_qsl(url_parts[4])) | trackers

--- a/frappe/utils/password.py
+++ b/frappe/utils/password.py
@ -215,4 +215,4 @@ def get_encryption_key():


 def get_password_reset_limit():
-	return frappe.db.get_single_value("System Settings", "password_reset_limit") or 0
+	return frappe.get_system_settings("password_reset_limit") or 3
--- a/frappe/utils/password_strength.py
+++ b/frappe/utils/password_strength.py
@ -50,9 +50,7 @@ default_feedback: "PasswordStrengthFeedback" = {
 def get_feedback(score: int, sequence: list) -> "PasswordStrengthFeedback":
 	"""Return the feedback dictionary consisting of ("warning","suggestions") for the given sequences."""
 	global default_feedback
-	minimum_password_score = int(
-		frappe.db.get_single_value("System Settings", "minimum_password_score") or 2
-	)
+	minimum_password_score = int(frappe.get_system_settings("minimum_password_score") or 2)

 	# Starting feedback
 	if len(sequence) == 0:
--- a/frappe/utils/pdf.py
+++ b/frappe/utils/pdf.py
@ -227,7 +227,7 @@ def read_options_from_html(html):
 	return str(soup), options


-def prepare_header_footer(soup):
+def prepare_header_footer(soup: BeautifulSoup):
 	options = {}

 	head = soup.find("head").contents
@ -238,9 +238,11 @@ def prepare_header_footer(soup):

 	# extract header and footer
 	for html_id in ("header-html", "footer-html"):
-		content = soup.find(id=html_id)
-		if content:
-			# there could be multiple instances of header-html/footer-html
+		if content := soup.find(id=html_id):
+			content = content.extract()
+			# `header/footer-html` are extracted, rendered as html
+			# and passed in wkhtmltopdf options (as '--header/footer-html')
+			# Remove instances of them from main content for render_template
 			for tag in soup.find_all(id=html_id):
 				tag.extract()

--- a/frappe/utils/response.py
+++ b/frappe/utils/response.py
@ -265,7 +265,15 @@ def download_backup(path):
 def download_private_file(path: str) -> Response:
 	"""Checks permissions and sends back private file"""

-	files = frappe.get_all("File", filters={"file_url": path}, fields="*")
+	if frappe.session.user == "Guest":
+		raise Forbidden(_("You don't have permission to access this file"))
+
+	filters = {"file_url": path}
+	if frappe.form_dict.fid:
+		filters["name"] = str(frappe.form_dict.fid)
+
+	files = frappe.get_all("File", filters=filters, fields="*")
+
 	# this file might be attached to multiple documents
 	# if the file is accessible from any one of those documents
 	# then it should be downloadable
--- a/Show more
+++ b/Show more