Merge remote-tracking branch 'upstream/develop' into widget-translatability

.github/workflows/server-tests.yml

@@ -43,6 +43,8 @@ jobs:
     needs: checkrun
     if: ${{ needs.checkrun.outputs.build == 'strawberry' }}
     timeout-minutes: 60
+    env:
+      NODE_ENV: "production"
 
     strategy:
       fail-fast: false

.github/workflows/ui-tests.yml

@@ -42,6 +42,8 @@ jobs:
     needs: checkrun
     if: ${{ needs.checkrun.outputs.build == 'strawberry' && github.repository_owner == 'frappe' }}
     timeout-minutes: 60
+    env:
+      NODE_ENV: "production"
 
     strategy:
       fail-fast: false

babel_extractors.csv

@@ -6,4 +6,4 @@ hooks.py,frappe.gettext.extractors.navbar.extract
 **/report/*/*.json,frappe.gettext.extractors.report.extract
 **.py,frappe.gettext.extractors.python.extract
 **.js,frappe.gettext.extractors.javascript.extract
-**.html,frappe.gettext.extractors.jinja2.extract
+**.html,frappe.gettext.extractors.html_template.extract

frappe/__init__.py

@@ -130,9 +130,45 @@ def _lt(msg: str, lang: str | None = None, context: str | None = None):
 
 	Note: Result is not guaranteed to equivalent to pure strings for all operations.
 	"""
-	from frappe.translate import LazyTranslate
+	return _LazyTranslate(msg, lang, context)
 
-	return LazyTranslate(msg, lang, context)
+
+@functools.total_ordering
+class _LazyTranslate:
+	__slots__ = ("msg", "lang", "context")
+
+	def __init__(self, msg: str, lang: str | None = None, context: str | None = None) -> None:
+		self.msg = msg
+		self.lang = lang
+		self.context = context
+
+	@property
+	def value(self) -> str:
+		return _(str(self.msg), self.lang, self.context)
+
+	def __str__(self):
+		return self.value
+
+	def __add__(self, other):
+		if isinstance(other, (str, _LazyTranslate)):
+			return self.value + str(other)
+		raise NotImplementedError
+
+	def __radd__(self, other):
+		if isinstance(other, (str, _LazyTranslate)):
+			return str(other) + self.value
+		return NotImplementedError
+
+	def __repr__(self) -> str:
+		return f"'{self.value}'"
+
+	# NOTE: it's required to override these methods and raise error as default behaviour will
+	# return `False` in all cases.
+	def __eq__(self, other):
+		raise NotImplementedError
+
+	def __lt__(self, other):
+		raise NotImplementedError
 
 
 def as_unicode(text, encoding: str = "utf-8") -> str:
@@ -269,10 +305,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False)
 
 	local.initialised = True
 
-	# Set the user as database name if not set in config
-	if local.conf and local.conf.db_name is not None and local.conf.db_user is None:
-		local.conf.db_user = local.conf.db_name
-
 
 def connect(
 	site: str | None = None, db_name: str | None = None, set_admin_as_user: bool = True
@@ -280,7 +312,7 @@ def connect(
 	"""Connect to site database instance.
 
 	:param site: (Deprecated) If site is given, calls `frappe.init`.
-	:param db_name: Optional. Will use from `site_config.json`.
+	:param db_name: (Deprecated) Optional. Will use from `site_config.json`.
 	:param set_admin_as_user: Set Administrator as current user.
 	"""
 	from frappe.database import get_db
@@ -293,13 +325,24 @@ def connect(
 			"Instead, explicitly invoke frappe.init(site) prior to calling frappe.connect(), if initializing the site is necessary."
 		)
 		init(site)
+	if db_name:
+		from frappe.utils.deprecations import deprecation_warning
+
+		deprecation_warning(
+			"Calling frappe.connect with the db_name argument is deprecated and will be removed in next major version. "
+			"Instead, explicitly invoke frappe.init(site) with the right config prior to calling frappe.connect(), if necessary."
+		)
+
+	assert db_name or local.conf.db_user, "site must be fully initialized, db_user missing"
+	assert db_name or local.conf.db_name, "site must be fully initialized, db_name missing"
+	assert local.conf.db_password, "site must be fully initialized, db_password missing"
 
 	local.db = get_db(
 		host=local.conf.db_host,
 		port=local.conf.db_port,
-		user=local.conf.db_user or db_name or local.conf.db_name,
+		user=local.conf.db_user or db_name,
 		password=local.conf.db_password,
-		cur_db_name=db_name or local.conf.db_name,
+		cur_db_name=local.conf.db_name or db_name,
 	)
 	if set_admin_as_user:
 		set_user("Administrator")
@@ -387,6 +430,11 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 		os.environ.get("FRAPPE_DB_PORT") or config.get("db_port") or db_default_ports(config["db_type"])
 	)
 
+	# Set the user as database name if not set in config
+	config["db_user"] = (
+		os.environ.get("FRAPPE_DB_USER") or config.get("db_user") or config.get("db_name")
+	)
+
 	return config
 
 

frappe/auth.py

@@ -274,9 +274,7 @@ class LoginManager:
 		if self.user in frappe.STANDARD_USERS:
 			return False
 
-		reset_pwd_after_days = cint(
-			frappe.db.get_single_value("System Settings", "force_user_to_reset_password")
-		)
+		reset_pwd_after_days = cint(frappe.get_system_settings("force_user_to_reset_password"))
 
 		if reset_pwd_after_days:
 			last_password_reset_date = (

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -550,7 +550,7 @@ def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters
 	docs += [r.name for r in res]
 	docs = set(list(docs))
 
-	return [[d] for d in docs]
+	return [[d] for d in docs if txt in d]
 
 
 @frappe.whitelist()

frappe/commands/utils.py

@@ -772,12 +772,8 @@ def run_tests(
 			click.secho(f"bench --site {site} set-config allow_tests true", fg="green")
 			return
 
-		frappe.init(site=site)
-
-		frappe.flags.skip_before_tests = skip_before_tests
-		frappe.flags.skip_test_records = skip_test_records
-
 		ret = frappe.test_runner.main(
+			site,
 			app,
 			module,
 			doctype,
@@ -790,6 +786,8 @@ def run_tests(
 			doctype_list_path=doctype_list_path,
 			failfast=failfast,
 			case=case,
+			skip_test_records=skip_test_records,
+			skip_before_tests=skip_before_tests,
 		)
 
 		if len(ret.failures) == 0 and len(ret.errors) == 0:

frappe/contacts/doctype/address_template/address_template.py

@@ -28,7 +28,7 @@ class AddressTemplate(Document):
 
 		if not self.is_default and not self._get_previous_default():
 			self.is_default = 1
-			if frappe.db.get_single_value("System Settings", "setup_complete"):
+			if frappe.get_system_settings("setup_complete"):
 				frappe.msgprint(_("Setting this Address Template as default as there is no other default"))
 
 	def on_update(self):

frappe/core/doctype/communication/mixins.py

@@ -146,7 +146,7 @@ class CommunicationEmailMixin:
 		return get_formatted_email(self.mail_sender_fullname(), mail=self.mail_sender())
 
 	def get_content(self, print_format=None):
-		if print_format and frappe.db.get_single_value("System Settings", "attach_view_link"):
+		if print_format and frappe.get_system_settings("attach_view_link"):
 			return self.content + self.get_attach_link(print_format)
 		return self.content
 

frappe/core/doctype/data_export/data_export.js

@@ -153,7 +153,7 @@ const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {
 
 	const options = fields.map((df) => {
 		return {
-			label: df.label,
+			label: __(df.label),
 			value: df.fieldname,
 			danger: df.reqd,
 			checked: 1,
@@ -163,7 +163,7 @@ const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {
 	const multicheck_control = frappe.ui.form.make_control({
 		parent: parent_wrapper,
 		df: {
-			label: doctype,
+			label: __(doctype),
 			fieldname: doctype + "_fields",
 			fieldtype: "MultiCheck",
 			options: options,

frappe/core/doctype/data_import/data_import.js

@@ -135,34 +135,29 @@ frappe.ui.form.on("Data Import", {
 				let failed_records = cint(r.message.failed);
 				let total_records = cint(r.message.total_records);
 
-				if (!total_records) return;
-				let action, message;
-				if (frm.doc.import_type === "Insert New Records") {
-					action = "imported";
-				} else {
-					action = "updated";
+				if (!total_records) {
+					return;
 				}
 
-				if (failed_records === 0) {
-					let message_args = [action, successful_records];
-					if (successful_records === 1) {
-						message = __("Successfully {0} 1 record.", message_args);
-					} else {
-						message = __("Successfully {0} {1} records.", message_args);
-					}
+				let message;
+				if (frm.doc.import_type === "Insert New Records") {
+					message = __("Successfully imported {0} out of {1} records.", [
+						successful_records,
+						total_records,
+					]);
 				} else {
-					let message_args = [action, successful_records, total_records];
-					if (successful_records === 1) {
-						message = __(
-							"Successfully {0} {1} record out of {2}. Click on Export Errored Rows, fix the errors and import again.",
-							message_args
+					message = __("Successfully updated {0} out of {1} records.", [
+						successful_records,
+						total_records,
+					]);
+				}
+
+				if (failed_records > 0) {
+					message +=
+						"<br/>" +
+						__(
+							"Please click on 'Export Errored Rows', fix the errors and import again."
 						);
-					} else {
-						message = __(
-							"Successfully {0} {1} records out of {2}. Click on Export Errored Rows, fix the errors and import again.",
-							message_args
-						);
-					}
 				}
 
 				// If the job timed out, display an extra hint
@@ -506,13 +501,7 @@ frappe.ui.form.on("Data Import", {
 	},
 
 	show_import_log(frm) {
-		if (!frm.doc.show_failed_logs) {
-			frm.toggle_display("import_log_preview", false);
-			return;
-		}
-
 		frm.toggle_display("import_log_section", false);
-		frm.toggle_display("import_log_preview", true);
 
 		if (frm.import_in_progress) {
 			return;

frappe/core/doctype/data_import/data_import.json

@@ -139,7 +139,7 @@
    "default": "0",
    "fieldname": "show_failed_logs",
    "fieldtype": "Check",
-   "label": "Show Failed Logs"
+   "label": "Show Only Failed Logs"
   },
   {
    "depends_on": "eval:!doc.__islocal && !doc.import_file",
@@ -171,7 +171,7 @@
  ],
  "hide_toolbar": 1,
  "links": [],
- "modified": "2023-12-15 12:45:49.452834",
+ "modified": "2024-01-30 17:08:05.566686",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Data Import",

frappe/core/doctype/data_import/data_import.py

@@ -38,7 +38,6 @@ class DataImport(Document):
 		submit_after_import: DF.Check
 		template_options: DF.Code | None
 		template_warnings: DF.Code | None
-
 	# end: auto-generated types
 
 	def validate(self):
@@ -93,7 +92,8 @@ class DataImport(Document):
 	def start_import(self):
 		from frappe.utils.scheduler import is_scheduler_inactive
 
-		if is_scheduler_inactive() and not frappe.flags.in_test:
+		run_now = frappe.flags.in_test or frappe.conf.developer_mode
+		if is_scheduler_inactive() and not run_now:
 			frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive"))
 
 		job_id = f"data_import::{self.name}"
@@ -106,7 +106,7 @@ class DataImport(Document):
 				event="data_import",
 				job_id=job_id,
 				data_import=self.name,
-				now=frappe.conf.developer_mode or frappe.flags.in_test,
+				now=run_now,
 			)
 			return True
 

frappe/core/doctype/docfield/docfield.json

@@ -157,6 +157,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval:!doc.is_virtual",
    "fieldname": "in_list_view",
    "fieldtype": "Check",
    "label": "In List View",
@@ -580,7 +581,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-11-16 11:26:56.364594",
+ "modified": "2024-02-01 15:55:44.007917",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocField",

frappe/core/doctype/docfield/docfield.py

@@ -117,6 +117,7 @@ class DocField(Document):
 		unique: DF.Check
 		width: DF.Data | None
 	# end: auto-generated types
+
 	def get_link_doctype(self):
 		"""Return the Link doctype for the `docfield` (if applicable).
 

frappe/core/doctype/file/file.py

@@ -756,6 +756,13 @@ class File(Document):
 		self.save_file(content=optimized_content, overwrite=True)
 		self.save()
 
+	@property
+	def unique_url(self) -> str:
+		"""Unique URL contains file ID in URL to speed up permisison checks."""
+		from urllib.parse import urlencode
+
+		return self.file_url + "?" + urlencode({"fid": self.name})
+
 	@staticmethod
 	def zip_files(files):
 		zip_file = io.BytesIO()

frappe/core/doctype/role_profile/role_profile.py

@@ -25,7 +25,11 @@ class RoleProfile(Document):
 		self.name = self.role_profile
 
 	def on_update(self):
-		self.queue_action("update_all_users", now=frappe.flags.in_test, enqueue_after_commit=True)
+		self.queue_action(
+			"update_all_users",
+			now=frappe.flags.in_test or frappe.flags.in_install,
+			enqueue_after_commit=True,
+		)
 
 	def update_all_users(self):
 		"""Changes in role_profile reflected across all its user"""

frappe/core/doctype/server_script/test_server_script.py

@@ -238,7 +238,6 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
 		script.execute_method()
 
 	def test_server_script_rate_limiting(self):
-		# why not
 		script1 = frappe.get_doc(
 			doctype="Server Script",
 			name="rate_limited_server_script",

frappe/core/doctype/system_settings/system_settings.py

@@ -132,6 +132,9 @@ class SystemSettings(Document):
 		self.validate_backup_limit()
 		self.validate_file_extensions()
 
+		if not self.link_field_results_limit:
+			self.link_field_results_limit = 10
+
 		if self.link_field_results_limit > 50:
 			self.link_field_results_limit = 50
 			label = _(self.meta.get_label("link_field_results_limit"))

frappe/core/doctype/user/user.py

@@ -725,12 +725,8 @@ class User(Document):
 		3. If allow_login_using_user_name is set, you can use username while finding the user.
 		"""
 
-		login_with_mobile = cint(
-			frappe.db.get_single_value("System Settings", "allow_login_using_mobile_number")
-		)
-		login_with_username = cint(
-			frappe.db.get_single_value("System Settings", "allow_login_using_user_name")
-		)
+		login_with_mobile = cint(frappe.get_system_settings("allow_login_using_mobile_number"))
+		login_with_username = cint(frappe.get_system_settings("allow_login_using_user_name"))
 
 		or_filters = [{"name": user_name}]
 		if login_with_mobile:
@@ -840,8 +836,8 @@ def update_password(
 	else:
 		user = res["user"]
 
-	logout_all_sessions = cint(logout_all_sessions) or frappe.db.get_single_value(
-		"System Settings", "logout_on_password_reset"
+	logout_all_sessions = cint(logout_all_sessions) or frappe.get_system_settings(
+		"logout_on_password_reset"
 	)
 	_update_password(user, new_password, logout_all_sessions=cint(logout_all_sessions))
 
@@ -933,7 +929,7 @@ def _get_user_for_update_password(key, old_password):
 		result.user, last_reset_password_key_generated_on = user or (None, None)
 		if result.user:
 			reset_password_link_expiry = cint(
-				frappe.db.get_single_value("System Settings", "reset_password_link_expiry_duration")
+				frappe.get_system_settings("reset_password_link_expiry_duration")
 			)
 			if (
 				reset_password_link_expiry
@@ -1018,7 +1014,7 @@ def sign_up(email: str, full_name: str, redirect_to: str) -> tuple[int, str]:
 
 
 @frappe.whitelist(allow_guest=True)
-@rate_limit(limit=get_password_reset_limit, seconds=24 * 60 * 60)
+@rate_limit(limit=get_password_reset_limit, seconds=60 * 60)
 def reset_password(user: str) -> str:
 	if user == "Administrator":
 		return "not allowed"
@@ -1254,34 +1250,37 @@ def create_contact(user, ignore_links=False, ignore_mandatory=False):
 		except frappe.DuplicateEntryError:
 			pass
 	else:
-		contact = frappe.get_doc("Contact", contact_name)
-		contact.first_name = user.first_name
-		contact.last_name = user.last_name
-		contact.gender = user.gender
+		try:
+			contact = frappe.get_doc("Contact", contact_name)
+			contact.first_name = user.first_name
+			contact.last_name = user.last_name
+			contact.gender = user.gender
 
-		# Add mobile number if phone does not exists in contact
-		if user.phone and not any(new_contact.phone == user.phone for new_contact in contact.phone_nos):
-			# Set primary phone if there is no primary phone number
-			contact.add_phone(
-				user.phone,
-				is_primary_phone=not any(
-					new_contact.is_primary_phone == 1 for new_contact in contact.phone_nos
-				),
-			)
+			# Add mobile number if phone does not exists in contact
+			if user.phone and not any(new_contact.phone == user.phone for new_contact in contact.phone_nos):
+				# Set primary phone if there is no primary phone number
+				contact.add_phone(
+					user.phone,
+					is_primary_phone=not any(
+						new_contact.is_primary_phone == 1 for new_contact in contact.phone_nos
+					),
+				)
 
-		# Add mobile number if mobile does not exists in contact
-		if user.mobile_no and not any(
-			new_contact.phone == user.mobile_no for new_contact in contact.phone_nos
-		):
-			# Set primary mobile if there is no primary mobile number
-			contact.add_phone(
-				user.mobile_no,
-				is_primary_mobile_no=not any(
-					new_contact.is_primary_mobile_no == 1 for new_contact in contact.phone_nos
-				),
-			)
+			# Add mobile number if mobile does not exists in contact
+			if user.mobile_no and not any(
+				new_contact.phone == user.mobile_no for new_contact in contact.phone_nos
+			):
+				# Set primary mobile if there is no primary mobile number
+				contact.add_phone(
+					user.mobile_no,
+					is_primary_mobile_no=not any(
+						new_contact.is_primary_mobile_no == 1 for new_contact in contact.phone_nos
+					),
+				)
 
-		contact.save(ignore_permissions=True)
+			contact.save(ignore_permissions=True)
+		except frappe.TimestampMismatchError:
+			raise frappe.RetryBackgroundJobError
 
 
 def get_restricted_ip_list(user):

frappe/custom/doctype/custom_field/custom_field.js

@@ -112,28 +112,30 @@ frappe.ui.form.on("Custom Field", {
 		}
 	},
 	add_rename_field(frm) {
-		frm.add_custom_button(__("Rename Fieldname"), () => {
-			frappe.prompt(
-				{
-					fieldtype: "Data",
-					label: __("Fieldname"),
-					fieldname: "fieldname",
-					reqd: 1,
-					default: frm.doc.fieldname,
-				},
-				function (data) {
-					frappe.call({
-						method: "frappe.custom.doctype.custom_field.custom_field.rename_fieldname",
-						args: {
-							custom_field: frm.doc.name,
-							fieldname: data.fieldname,
-						},
-					});
-				},
-				__("Rename Fieldname"),
-				__("Rename")
-			);
-		});
+		if (!frm.is_new()) {
+			frm.add_custom_button(__("Rename Fieldname"), () => {
+				frappe.prompt(
+					{
+						fieldtype: "Data",
+						label: __("Fieldname"),
+						fieldname: "fieldname",
+						reqd: 1,
+						default: frm.doc.fieldname,
+					},
+					function (data) {
+						frappe.call({
+							method: "frappe.custom.doctype.custom_field.custom_field.rename_fieldname",
+							args: {
+								custom_field: frm.doc.name,
+								fieldname: data.fieldname,
+							},
+						});
+					},
+					__("Rename Fieldname"),
+					__("Rename")
+				);
+			});
+		}
 	},
 });
 

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -130,6 +130,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval:!doc.is_virtual",
    "fieldname": "in_list_view",
    "fieldtype": "Check",
    "label": "In List View"
@@ -483,7 +484,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-12-08 15:52:37.525003",
+ "modified": "2024-02-01 15:56:39.171633",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",

frappe/custom/doctype/customize_form_field/customize_form_field.py

@@ -110,4 +110,5 @@ class CustomizeFormField(Document):
 		unique: DF.Check
 		width: DF.Data | None
 	# end: auto-generated types
+
 	pass

frappe/database/__init__.py

@@ -23,17 +23,17 @@ def setup_database(force, verbose=None, no_mariadb_socket=False):
 		)
 
 
-def bootstrap_database(db_name, verbose=None, source_sql=None):
+def bootstrap_database(verbose=None, source_sql=None):
 	import frappe
 
 	if frappe.conf.db_type == "postgres":
 		import frappe.database.postgres.setup_db
 
-		return frappe.database.postgres.setup_db.bootstrap_database(db_name, verbose, source_sql)
+		return frappe.database.postgres.setup_db.bootstrap_database(verbose, source_sql)
 	else:
 		import frappe.database.mariadb.setup_db
 
-		return frappe.database.mariadb.setup_db.bootstrap_database(db_name, verbose, source_sql)
+		return frappe.database.mariadb.setup_db.bootstrap_database(verbose, source_sql)
 
 
 def drop_user_and_database(db_name, db_user):
@@ -75,12 +75,7 @@ def get_command(
 		else:
 			bin, bin_name = which("psql"), "psql"
 
-		host = frappe.utils.esc(host, "$ ")
-		user = frappe.utils.esc(user, "$ ")
-		db_name = frappe.utils.esc(db_name, "$ ")
-
 		if password:
-			password = frappe.utils.esc(password, "$ ")
 			conn_string = f"postgresql://{user}:{password}@{host}:{port}/{db_name}"
 		else:
 			conn_string = f"postgresql://{user}@{host}:{port}/{db_name}"
@@ -96,10 +91,6 @@ def get_command(
 		else:
 			bin, bin_name = which("mariadb") or which("mysql"), "mariadb"
 
-		host = frappe.utils.esc(host, "$ ")
-		user = frappe.utils.esc(user, "$ ")
-		db_name = frappe.utils.esc(db_name, "$ ")
-
 		command = [
 			f"--user={user}",
 			f"--host={host}",
@@ -107,7 +98,6 @@ def get_command(
 		]
 
 		if password:
-			password = frappe.utils.esc(password, "$ ")
 			command.append(f"--password={password}")
 
 		if dump:

frappe/database/mariadb/setup_db.py

@@ -67,17 +67,17 @@ def drop_user_and_database(
 	dbman.delete_user(db_user)
 
 
-def bootstrap_database(db_name, verbose, source_sql=None):
+def bootstrap_database(verbose, source_sql=None):
 	import sys
 
-	frappe.connect(db_name=db_name)
+	frappe.connect()
 	if not check_database_settings():
 		print("Database settings do not match expected values; stopping database setup.")
 		sys.exit(1)
 
 	import_db_from_sql(source_sql, verbose)
-	frappe.connect(db_name=db_name)
 
+	frappe.connect()
 	if "tabDefaultValue" not in frappe.db.get_tables(cached=False):
 		from click import secho
 

frappe/database/postgres/setup_db.py

@@ -29,11 +29,12 @@ def setup_database():
 	root_conn.close()
 
 
-def bootstrap_database(db_name, verbose, source_sql=None):
-	frappe.connect(db_name=db_name)
-	import_db_from_sql(source_sql, verbose)
-	frappe.connect(db_name=db_name)
+def bootstrap_database(verbose, source_sql=None):
 
+	frappe.connect()
+	import_db_from_sql(source_sql, verbose)
+
+	frappe.connect()
 	if "tabDefaultValue" not in frappe.db.get_tables():
 		import sys
 

frappe/desk/doctype/note/note.py

@@ -28,6 +28,9 @@ class Note(Document):
 			# expire this notification in a week (default)
 			self.expire_notification_on = frappe.utils.add_days(self.creation, 7)
 
+		if not self.public and self.notify_on_login:
+			self.notify_on_login = 0
+
 		if not self.content:
 			self.content = "<span></span>"
 

frappe/desk/form/test_form.py

@@ -11,10 +11,3 @@ class TestForm(FrappeTestCase):
 		results = get_linked_docs("Role", "System Manager", linkinfo=get_linked_doctypes("Role"))
 		self.assertTrue("User" in results)
 		self.assertTrue("DocType" in results)
-
-
-if __name__ == "__main__":
-	import unittest
-
-	frappe.connect()
-	unittest.main()

frappe/desk/notifications.py

@@ -25,7 +25,7 @@ def get_notifications():
 		"open_count_doctype": {},
 		"targets": {},
 	}
-	if frappe.flags.in_install or not frappe.db.get_single_value("System Settings", "setup_complete"):
+	if frappe.flags.in_install or not frappe.get_system_settings("setup_complete"):
 		return out
 
 	config = get_notification_config()

frappe/email/doctype/auto_email_report/auto_email_report.json

@@ -25,6 +25,7 @@
   "to_date_field",
   "column_break_17",
   "dynamic_date_period",
+  "use_first_day_of_period",
   "email_settings",
   "email_to",
   "day_of_week",
@@ -207,10 +208,18 @@
    "fieldtype": "Link",
    "label": "Sender",
    "options": "Email Account"
+  },
+  {
+   "default": "0",
+   "description": "To begin the date range at the start of the chosen period. For example, if 'Year' is selected as the period, the report will start from January 1st of the current year.",
+   "fieldname": "use_first_day_of_period",
+   "fieldtype": "Check",
+   "depends_on": "eval: doc.dynamic_date_period != 'Daily'",
+   "label": "Use First Day of Period"
   }
  ],
  "links": [],
- "modified": "2022-09-08 15:31:55.031023",
+ "modified": "2024-01-29 11:42:27.433958",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Auto Email Report",
@@ -245,4 +254,4 @@
  "sort_order": "DESC",
  "states": [],
  "track_changes": 1
-}
+}

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -2,6 +2,7 @@
 # License: MIT. See LICENSE
 
 import calendar
+import datetime
 from datetime import timedelta
 from email.utils import formataddr
 
@@ -14,8 +15,13 @@ from frappe.utils import (
 	add_to_date,
 	cint,
 	format_time,
+	get_first_day,
+	get_first_day_of_week,
 	get_link_to_form,
+	get_quarter_start,
 	get_url_to_report,
+	get_year_start,
+	getdate,
 	global_date_format,
 	now,
 	now_datetime,
@@ -57,8 +63,10 @@ class AutoEmailReport(Document):
 		send_if_data: DF.Check
 		sender: DF.Link | None
 		to_date_field: DF.Literal
+		use_first_day_of_period: DF.Check
 		user: DF.Link
 	# end: auto-generated types
+
 	def autoname(self):
 		self.name = _(self.report)
 		if frappe.db.exists("Auto Email Report", self.name):
@@ -92,7 +100,7 @@ class AutoEmailReport(Document):
 
 		max_reports_per_user = (
 			cint(frappe.local.conf.max_reports_per_user)  # kept for backward compatibilty
-			or cint(frappe.db.get_single_value("System Settings", "max_auto_email_report_per_user"))
+			or cint(frappe.get_system_settings("max_auto_email_report_per_user"))
 			or 20
 		)
 
@@ -207,17 +215,37 @@ class AutoEmailReport(Document):
 		self.filters = frappe.parse_json(self.filters)
 
 		to_date = today()
-		from_date_value = {
-			"Daily": ("days", -1),
-			"Weekly": ("weeks", -1),
-			"Monthly": ("months", -1),
-			"Quarterly": ("months", -3),
-			"Half Yearly": ("months", -6),
-			"Yearly": ("years", -1),
-		}[self.dynamic_date_period]
 
-		from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
+		if self.use_first_day_of_period:
+			from_date = to_date
+			if self.dynamic_date_period == "Daily":
+				from_date = add_to_date(to_date, days=-1)
+			elif self.dynamic_date_period == "Weekly":
+				from_date = get_first_day_of_week(from_date)
+			elif self.dynamic_date_period == "Monthly":
+				from_date = get_first_day(from_date)
+			elif self.dynamic_date_period == "Quarterly":
+				from_date = get_quarter_start(from_date)
+			elif self.dynamic_date_period == "Half Yearly":
+				from_date = get_half_year_start(from_date)
+			elif self.dynamic_date_period == "Yearly":
+				from_date = get_year_start(from_date)
 
+			self.set_date_filters(from_date, to_date)
+		else:
+			from_date_value = {
+				"Daily": ("days", -1),
+				"Weekly": ("weeks", -1),
+				"Monthly": ("months", -1),
+				"Quarterly": ("months", -3),
+				"Half Yearly": ("months", -6),
+				"Yearly": ("years", -1),
+			}[self.dynamic_date_period]
+
+			from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
+			self.set_date_filters(from_date, to_date)
+
+	def set_date_filters(self, from_date, to_date):
 		self.filters[self.from_date_field] = from_date
 		self.filters[self.to_date_field] = to_date
 
@@ -332,3 +360,23 @@ def update_field_types(columns):
 			col.fieldtype = "Data"
 			col.options = ""
 	return columns
+
+
+DATE_FORMAT = "%Y-%m-%d"
+
+
+def get_half_year_start(as_str=False):
+	"""
+	Returns the first day of the current half-year based on the current date.
+	"""
+	today_date = getdate(today())
+
+	half_year = 1 if today_date.month <= 6 else 2
+
+	year = today_date.year if half_year == 1 else today_date.year + 1
+	month = 1 if half_year == 1 else 7
+	day = 1
+
+	result_date = datetime.date(year, month, day)
+
+	return result_date if not as_str else result_date.strftime(DATE_FORMAT)

frappe/email/doctype/email_queue/email_queue.py

@@ -131,12 +131,12 @@ class EmailQueue(Document):
 	def attachments_list(self):
 		return json.loads(self.attachments) if self.attachments else []
 
-	def get_email_account(self):
+	def get_email_account(self, raise_error=False):
 		if self.email_account:
 			return frappe.get_cached_doc("Email Account", self.email_account)
 
 		return EmailAccount.find_outgoing(
-			match_by_email=self.sender, match_by_doctype=self.reference_doctype
+			match_by_email=self.sender, match_by_doctype=self.reference_doctype, _raise_error=raise_error
 		)
 
 	def is_to_be_sent(self):
@@ -158,6 +158,7 @@ class EmailQueue(Document):
 			return
 
 		with SendMailContext(self, smtp_server_instance) as ctx:
+			ctx.fetch_smtp_server()
 			message = None
 			for recipient in self.recipients:
 				if recipient.is_mail_sent():
@@ -233,14 +234,16 @@ class SendMailContext:
 		smtp_server_instance: SMTPServer = None,
 	):
 		self.queue_doc: EmailQueue = queue_doc
-		self.email_account_doc = queue_doc.get_email_account()
-
-		self.smtp_server: SMTPServer = smtp_server_instance or self.email_account_doc.get_smtp_server()
-
+		self.smtp_server: SMTPServer = smtp_server_instance
 		self.sent_to_atleast_one_recipient = any(
 			rec.recipient for rec in self.queue_doc.recipients if rec.is_mail_sent()
 		)
 
+	def fetch_smtp_server(self):
+		self.email_account_doc = self.queue_doc.get_email_account(raise_error=True)
+		if not self.smtp_server:
+			self.smtp_server = self.email_account_doc.get_smtp_server()
+
 	def __enter__(self):
 		self.queue_doc.update_status(status="Sending", commit=True)
 		return self
@@ -733,7 +736,7 @@ class QueueBuilder:
 			recipients = list(set([r] + self.final_cc() + self.bcc))
 			q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True)
 			if not smtp_server_instance:
-				email_account = q.get_email_account()
+				email_account = q.get_email_account(raise_error=True)
 				smtp_server_instance = email_account.get_smtp_server()
 
 			with suppress(Exception):

frappe/email/doctype/newsletter/newsletter.json

@@ -3,7 +3,7 @@
  "allow_guest_to_view": 1,
  "allow_rename": 1,
  "creation": "2013-01-10 16:34:31",
- "description": "Create and Send Newsletters",
+ "description": "Create and send emails to a specific group of subscribers periodically.",
  "doctype": "DocType",
  "document_type": "Other",
  "engine": "InnoDB",
@@ -244,8 +244,7 @@
    "fieldname": "campaign",
    "fieldtype": "Link",
    "label": "Campaign",
-   "options": "Marketing Campaign",
-   "reqd": 0
+   "options": "Marketing Campaign"
   }
  ],
  "has_web_view": 1,
@@ -254,7 +253,7 @@
  "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2023-12-29 18:04:13.270523",
+ "modified": "2024-01-30 14:05:50.645802",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",

frappe/email/receive.py

@@ -673,7 +673,7 @@ class InboundMail(Email):
 		content = self.content
 		for file in attachments:
 			if file.name in self.cid_map and self.cid_map[file.name]:
-				content = content.replace(f"cid:{self.cid_map[file.name]}", file.file_url)
+				content = content.replace(f"cid:{self.cid_map[file.name]}", file.unique_url)
 		return content
 
 	def is_notification(self):

frappe/geo/doctype/currency/currency.json

@@ -4,7 +4,7 @@
  "allow_rename": 1,
  "autoname": "field:currency_name",
  "creation": "2013-01-28 10:06:02",
- "description": "**Currency** Master",
+ "description": "Currency list stores the currency value, its symbol and fraction unit",
  "doctype": "DocType",
  "document_type": "Setup",
  "engine": "InnoDB",
@@ -82,7 +82,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2024-01-17 15:37:31.605278",
+ "modified": "2024-01-30 13:18:12.053557",
  "modified_by": "Administrator",
  "module": "Geo",
  "name": "Currency",

frappe/gettext/extractors/html_template.py

@@ -0,0 +1,26 @@
+from jinja2.ext import babel_extract
+
+from .utils import extract_messages_from_code
+
+
+def extract(*args, **kwargs):
+	"""Extract messages from Jinja and JS microtemplates.
+
+	Reuse the babel_extract function from jinja2.ext, but handle our own implementation of `_()`.
+	To handle JS microtemplates, parse all code again using regex."""
+	fileobj = args[0] or kwargs["fileobj"]
+	print(fileobj.name)
+	code = fileobj.read().decode("utf-8")
+
+	for lineno, funcname, messages, comments in babel_extract(*args, **kwargs):
+		if funcname == "_" and isinstance(messages, tuple) and len(messages) > 1:
+			funcname = "pgettext"
+			messages = (messages[-1], messages[0])  # (context, message)
+
+		yield lineno, funcname, messages, comments
+
+	for lineno, message, context in extract_messages_from_code(code):
+		if context:
+			yield lineno, "pgettext", (context, message), []
+		else:
+			yield lineno, "_", message, []

frappe/gettext/extractors/jinja2.py

@@ -1,11 +0,0 @@
-from jinja2.ext import babel_extract
-
-
-def extract(*args, **kwargs):
-	"""Reuse the babel_extract function from jinja2.ext, but handle our own implementation of `_()`"""
-	for lineno, funcname, messages, comments in babel_extract(*args, **kwargs):
-		if funcname == "_" and isinstance(messages, tuple) and len(messages) > 1:
-			funcname = "pgettext"
-			messages = (messages[-1], messages[0])  # (context, message)
-
-		yield lineno, funcname, messages, comments

frappe/gettext/extractors/utils.py

@@ -0,0 +1,81 @@
+import re
+
+import frappe
+
+TRANSLATE_PATTERN = re.compile(
+	r"_\(\s*"  # starts with literal `_(`, ignore following whitespace/newlines
+	# BEGIN: message search
+	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
+	r"(?P<message>((?!\1).)*)"  # Keep matching until string closing identifier is met which is same as 1st capture group
+	r"\1"  # match exact string closing identifier
+	# END: message search
+	# BEGIN: python context search
+	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
+	r"([\"'])"  # start of context string identifier; 5th capture group
+	r"(?P<py_context>((?!\5).)*)"  # capture context string till closing id is found
+	r"\5"  # match context string closure
+	r")?"  # match 0 or 1 context strings
+	# END: python context search
+	# BEGIN: JS context search
+	r"(\s*,\s*(.)*?\s*(,\s*"  # skip message format replacements: ["format", ...] | null | []
+	r"([\"'])"  # start of context string; 11th capture group
+	r"(?P<js_context>((?!\11).)*)"  # capture context string till closing id is found
+	r"\11"  # match context string closure
+	r")*"
+	r")*"  # match one or more context string
+	# END: JS context search
+	r"\s*\)"  # Closing function call ignore leading whitespace/newlines
+)
+
+
+def extract_messages_from_code(code):
+	"""
+	Extracts translatable strings from a code file
+	:param code: code from which translatable files are to be extracted
+	"""
+	from jinja2 import TemplateError
+
+	from frappe.model.utils import InvalidIncludePath, render_include
+
+	try:
+		code = frappe.as_unicode(render_include(code))
+
+	# Exception will occur when it encounters John Resig's microtemplating code
+	except (TemplateError, ImportError, InvalidIncludePath, OSError) as e:
+		if isinstance(e, InvalidIncludePath) and hasattr(frappe.local, "message_log"):
+			frappe.clear_last_message()
+
+	messages = []
+
+	for m in TRANSLATE_PATTERN.finditer(code):
+		message = m.group("message")
+		context = m.group("py_context") or m.group("js_context")
+		pos = m.start()
+
+		if is_translatable(message):
+			messages.append([pos, message, context])
+
+	return add_line_number(messages, code)
+
+
+def is_translatable(m):
+	return bool(
+		re.search("[a-zA-Z]", m)
+		and not m.startswith("fa fa-")
+		and not m.endswith("px")
+		and not m.startswith("eval:")
+	)
+
+
+def add_line_number(messages, code):
+	ret = []
+	messages = sorted(messages, key=lambda x: x[0])
+	newlines = [m.start() for m in re.compile(r"\n").finditer(code)]
+	line = 1
+	newline_i = 0
+	for pos, message, context in messages:
+		while newline_i < len(newlines) and pos > newlines[newline_i]:
+			line += 1
+			newline_i += 1
+		ret.append([line, message, context])
+	return ret

frappe/handler.py

@@ -250,7 +250,7 @@ def check_write_permission(doctype: str = None, name: str = None):
 	if doctype and name:
 		try:
 			doc = frappe.get_doc(doctype, name)
-			doc.has_permission("write")
+			doc.check_permission("write")
 		except frappe.DoesNotExistError:
 			# doc has not been inserted yet, name is set to "new-some-doctype"
 			check_doctype = True

frappe/hooks.py

@@ -532,15 +532,15 @@ standard_help_items = [
 
 # log doctype cleanups to automatically add in log settings
 default_log_clearing_doctypes = {
-	"Error Log": 30,
-	"Activity Log": 90,
+	"Error Log": 14,
 	"Email Queue": 30,
-	"Scheduled Job Log": 90,
-	"Route History": 90,
-	"Submission Queue": 30,
-	"Prepared Report": 30,
+	"Scheduled Job Log": 7,
+	"Submission Queue": 7,
+	"Prepared Report": 14,
 	"Webhook Request Log": 30,
-	"Integration Request": 90,
 	"Unhandled Email": 30,
 	"Reminder": 30,
+	"Integration Request": 90,
+	"Activity Log": 90,
+	"Route History": 90,
 }

frappe/installer.py

@@ -20,9 +20,10 @@ from frappe.utils.dashboard import sync_dashboards
 from frappe.utils.synchronization import filelock
 
 
-def _is_scheduler_enabled() -> bool:
+def _is_scheduler_enabled(site) -> bool:
 	enable_scheduler = False
 	try:
+		frappe.init(site=site)
 		frappe.connect()
 		enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler"))
 	except Exception:
@@ -78,7 +79,7 @@ def _new_site(
 
 	try:
 		# enable scheduler post install?
-		enable_scheduler = _is_scheduler_enabled()
+		enable_scheduler = _is_scheduler_enabled(site)
 	except Exception:
 		enable_scheduler = False
 
@@ -170,7 +171,6 @@ def install_db(
 		setup_database(force, verbose, no_mariadb_socket)
 
 	bootstrap_database(
-		db_name=frappe.conf.db_name,
 		verbose=verbose,
 		source_sql=source_sql,
 	)

frappe/integrations/doctype/google_calendar/google_calendar.py

@@ -239,7 +239,7 @@ def check_google_calendar(account, google_calendar):
 			# If no Calendar ID create a new Calendar
 			calendar = {
 				"summary": account.calendar_name,
-				"timeZone": frappe.db.get_single_value("System Settings", "time_zone"),
+				"timeZone": frappe.get_system_settings("time_zone"),
 			}
 			created_calendar = google_calendar.calendars().insert(body=calendar).execute()
 			frappe.db.set_value(

frappe/integrations/offsite_backup_utils.py

@@ -41,9 +41,6 @@ def send_email(success, service_name, doctype, email_field, error_status=None):
 
 
 def get_recipients(doctype, email_field):
-	if not frappe.db:
-		frappe.connect()
-
 	return split_emails(frappe.db.get_value(doctype, None, email_field))
 
 

frappe/model/__init__.py

@@ -134,22 +134,22 @@ log_types = (
 )
 
 std_fields = [
-	{"fieldname": "name", "fieldtype": "Link", "label": _lt("ID")},
-	{"fieldname": "owner", "fieldtype": "Link", "label": _lt("Created By"), "options": "User"},
-	{"fieldname": "idx", "fieldtype": "Int", "label": _lt("Index")},
-	{"fieldname": "creation", "fieldtype": "Datetime", "label": _lt("Created On")},
-	{"fieldname": "modified", "fieldtype": "Datetime", "label": _lt("Last Updated On")},
+	{"fieldname": "name", "fieldtype": "Link", "label": "ID"},
+	{"fieldname": "owner", "fieldtype": "Link", "label": "Created By", "options": "User"},
+	{"fieldname": "idx", "fieldtype": "Int", "label": "Index"},
+	{"fieldname": "creation", "fieldtype": "Datetime", "label": "Created On"},
+	{"fieldname": "modified", "fieldtype": "Datetime", "label": "Last Updated On"},
 	{
 		"fieldname": "modified_by",
 		"fieldtype": "Link",
-		"label": _lt("Last Updated By"),
+		"label": "Last Updated By",
 		"options": "User",
 	},
-	{"fieldname": "_user_tags", "fieldtype": "Data", "label": _lt("Tags")},
-	{"fieldname": "_liked_by", "fieldtype": "Data", "label": _lt("Liked By")},
-	{"fieldname": "_comments", "fieldtype": "Text", "label": _lt("Comments")},
-	{"fieldname": "_assign", "fieldtype": "Text", "label": _lt("Assigned To")},
-	{"fieldname": "docstatus", "fieldtype": "Int", "label": _lt("Document Status")},
+	{"fieldname": "_user_tags", "fieldtype": "Data", "label": "Tags"},
+	{"fieldname": "_liked_by", "fieldtype": "Data", "label": "Liked By"},
+	{"fieldname": "_comments", "fieldtype": "Text", "label": "Comments"},
+	{"fieldname": "_assign", "fieldtype": "Text", "label": "Assigned To"},
+	{"fieldname": "docstatus", "fieldtype": "Int", "label": "Document Status"},
 ]
 
 

frappe/model/delete_doc.py

@@ -133,6 +133,7 @@ def delete_doc(
 					doctype=doc.doctype,
 					name=doc.name,
 					now=frappe.flags.in_test,
+					enqueue_after_commit=True,
 				)
 
 		# clear cache for Document

frappe/model/utils/__init__.py

@@ -66,7 +66,7 @@ def render_include(content):
 		if "{% include" in content:
 			paths = INCLUDE_DIRECTIVE_PATTERN.findall(content)
 			if not paths:
-				frappe.throw(_("Invalid include path"), InvalidIncludePath)
+				raise InvalidIncludePath
 
 			for path in paths:
 				app, app_path = path.split("/", 1)

frappe/printing/doctype/letter_head/letter_head.js

@@ -2,7 +2,60 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Letter Head", {
+	setup(frm) {
+		frm.get_field("instructions").html(INSTRUCTIONS);
+	},
+
 	refresh: function (frm) {
 		frm.flag_public_attachments = true;
 	},
+
+	validate: (frm) => {
+		["header_script", "footer_script"].forEach((field) => {
+			if (!frm.doc[field]) return;
+
+			try {
+				eval(frm.doc[field]);
+			} catch (e) {
+				frappe.throw({
+					title: __("Error in Header/Footer Script"),
+					indicator: "orange",
+					message: '<pre class="small"><code>' + e.stack + "</code></pre>",
+				});
+			}
+		});
+	},
 });
+
+const INSTRUCTIONS = `<h4>${__("Letter Head Scripts")}</h4>
+<p>${__("Header/Footer scripts can be used to add dynamic behaviours.")}</p>
+<pre>
+<code>
+// ${__(
+	"The following Header Script will add the current date to an element in 'Header HTML' with class 'header-content'"
+)}
+var el = document.getElementsByClassName("header-content");
+if (el.length > 0) {
+	el[0].textContent += " " + new Date().toGMTString();
+}
+</code>
+</pre>
+<p>${__("You can also access wkhtmltopdf variables (valid only in PDF print):")}</p>
+<pre>
+<code>
+// ${__("Get Header and Footer wkhtmltopdf variables")}
+// ${__("Snippet and more variables:  {0}", ["https://wkhtmltopdf.org/usage/wkhtmltopdf.txt"])}
+var vars = {};
+var query_strings_from_url = document.location.search.substring(1).split('&');
+for (var query_string in query_strings_from_url) {
+	if (query_strings_from_url.hasOwnProperty(query_string)) {
+		var temp_var = query_strings_from_url[query_string].split('=', 2);
+		vars[temp_var[0]] = decodeURI(temp_var[1]);
+	}
+}
+var el = document.getElementsByClassName("header-content");
+if (el.length > 0 && vars["page"] == 1) {
+	el[0].textContent += " : " + vars["date"];
+}
+</code>
+</pre>`;

frappe/printing/doctype/letter_head/letter_head.json

@@ -26,7 +26,11 @@
   "footer_image",
   "footer_image_height",
   "footer_image_width",
-  "footer_align"
+  "footer_align",
+  "scripts_section",
+  "header_script",
+  "footer_script",
+  "instructions"
  ],
  "fields": [
   {
@@ -162,13 +166,40 @@
    "fieldtype": "Select",
    "label": "Footer Based On",
    "options": "Image\nHTML"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && doc.source==='HTML'",
+   "fieldname": "header_script",
+   "fieldtype": "Code",
+   "label": "Header Script",
+   "options": "Javascript"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && doc.footer_source==='HTML'",
+   "fieldname": "footer_script",
+   "fieldtype": "Code",
+   "label": "Footer Script",
+   "options": "Javascript"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "eval: doc.header_script || doc.footer_script",
+   "fieldname": "scripts_section",
+   "fieldtype": "Section Break",
+   "label": "Scripts"
+  },
+  {
+   "fieldname": "instructions",
+   "fieldtype": "HTML",
+   "label": "Instructions",
+   "read_only": 1
   }
  ],
  "icon": "fa fa-font",
  "idx": 1,
  "links": [],
  "max_attachments": 3,
- "modified": "2023-12-08 15:52:37.525003",
+ "modified": "2023-12-21 16:19:37.525003",
  "modified_by": "Administrator",
  "module": "Printing",
  "name": "Letter Head",

frappe/public/js/form_builder/store.js

@@ -85,8 +85,13 @@ export const useStore = defineStore("form-builder-store", () => {
 
 	async function fetch() {
 		doc.value = frm.value.doc;
-		if (doctype.value.startsWith("new-doctype-") && !doc.value.fields) {
-			doc.value.fields = [get_df("Data", "", __("Title"))];
+		if (doctype.value.startsWith("new-doctype-") && !doc.value.fields?.length) {
+			frappe.model.with_doctype("DocType").then(() => {
+				frappe.listview_settings["DocType"].new_doctype_dialog();
+			});
+			// redirect to /doctype
+			frappe.set_route("List", "DocType");
+			return;
 		}
 
 		if (!get_docfields.value.length) {

frappe/public/js/frappe/assets.js

@@ -19,11 +19,22 @@ frappe.require = function (items, callback) {
 	});
 };
 
-frappe.assets = {
-	check: function () {
+class AssetManager {
+	constructor() {
+		this._executed = [];
+		this._handlers = {
+			js: function (txt) {
+				frappe.dom.eval(txt);
+			},
+			css: function (txt) {
+				frappe.dom.set_style(txt);
+			},
+		};
+	}
+	check() {
 		// if version is different then clear localstorage
 		if (window._version_number != localStorage.getItem("_version_number")) {
-			frappe.assets.clear_local_storage();
+			this.clear_local_storage();
 			console.log("Cleared App Cache.");
 		}
 
@@ -33,160 +44,79 @@ frappe.assets = {
 			// Evict cache if page is reloaded within 10 seconds. Which could be user trying to
 			// refresh if things feel broken.
 			if ((not_updated_since < 5000 && is_reload()) || not_updated_since > 2 * 86400000) {
-				frappe.assets.clear_local_storage();
+				this.clear_local_storage();
 			}
 		} else {
-			frappe.assets.clear_local_storage();
+			this.clear_local_storage();
 		}
 
-		frappe.assets.init_local_storage();
-	},
+		this.init_local_storage();
+	}
 
-	init_local_storage: function () {
+	init_local_storage() {
 		localStorage._last_load = new Date();
 		localStorage._version_number = window._version_number;
 		if (frappe.boot) localStorage.metadata_version = frappe.boot.metadata_version;
-	},
+	}
 
-	clear_local_storage: function () {
-		$.each(
-			["_last_load", "_version_number", "metadata_version", "page_info", "last_visited"],
-			function (i, key) {
-				localStorage.removeItem(key);
-			}
+	clear_local_storage() {
+		["_last_load", "_version_number", "metadata_version", "page_info", "last_visited"].forEach(
+			(key) => localStorage.removeItem(key)
 		);
 
 		// clear assets
-		for (var key in localStorage) {
+		for (let key in localStorage) {
 			if (
-				key.indexOf("desk_assets:") === 0 ||
-				key.indexOf("_page:") === 0 ||
-				key.indexOf("_doctype:") === 0 ||
-				key.indexOf("preferred_breadcrumbs:") === 0
+				key.startsWith("_page:") ||
+				key.startsWith("_doctype:") ||
+				key.startsWith("preferred_breadcrumbs:")
 			) {
 				localStorage.removeItem(key);
 			}
 		}
 		console.log("localStorage cleared");
-	},
+	}
 
-	// keep track of executed assets
-	executed_: [],
+	eval_assets(path, content) {
+		if (!this._executed.includes(path)) {
+			this._handlers[this.extn(path)](content);
+			this._executed.push(path);
+		}
+	}
 
-	// pass on to the handler to set
-	execute: function (items, callback) {
-		var to_fetch = [];
-		for (var i = 0, l = items.length; i < l; i++) {
-			if (!frappe.assets.exists(items[i])) {
-				to_fetch.push(items[i]);
-			}
-		}
-		if (to_fetch.length) {
-			frappe.assets.fetch(to_fetch, function () {
-				frappe.assets.eval_assets(items, callback);
-			});
-		} else {
-			frappe.assets.eval_assets(items, callback);
-		}
-	},
-
-	eval_assets: function (items, callback) {
-		for (var i = 0, l = items.length; i < l; i++) {
-			// execute js/css if not already.
-			var path = items[i];
-			if (frappe.assets.executed_.indexOf(path) === -1) {
-				// execute
-				frappe.assets.handler[frappe.assets.extn(path)](frappe.assets.get(path), path);
-				frappe.assets.executed_.push(path);
-			}
-		}
-		callback && callback();
-	},
-
-	// check if the asset exists in
-	// localstorage
-	exists: function (src) {
-		if (frappe.assets.executed_.indexOf(src) !== -1) {
-			return true;
-		}
-		if (frappe.boot.developer_mode) {
-			return false;
-		}
-		if (frappe.assets.get(src)) {
-			return true;
-		} else {
-			return false;
-		}
-	},
-
-	// load an asset via
-	fetch: function (items, callback) {
+	execute(items, callback) {
 		// this is virtual page load, only get the the source
-		// *without* the template
-
-		if (items.length === 0) {
-			callback();
-			return;
-		}
+		let me = this;
 
 		const version_string =
 			frappe.boot.developer_mode || window.dev_server ? Date.now() : window._version_number;
 
-		async function fetch_item(item) {
+		async function fetch_item(path) {
 			// Add the version to the URL to bust the cache for non-bundled assets
-			let url = new URL(item, window.location.origin);
+			let url = new URL(path, window.location.origin);
 
-			if (!item.includes(".bundle.") && !url.searchParams.get("v")) {
+			if (!path.includes(".bundle.") && !url.searchParams.get("v")) {
 				url.searchParams.append("v", version_string);
 			}
 			const response = await fetch(url.toString());
 			const body = await response.text();
-			frappe.assets.add(item, body);
+			me.eval_assets(path, body);
 		}
 
 		frappe.dom.freeze();
 		const fetch_promises = items.map(fetch_item);
 		Promise.all(fetch_promises).then(() => {
 			frappe.dom.unfreeze();
-			callback();
+			callback?.();
 		});
-	},
+	}
 
-	add: function (src, txt) {
-		if ("localStorage" in window) {
-			try {
-				frappe.assets.set(src, txt);
-			} catch (e) {
-				// if quota is exceeded, clear local storage and set item
-				frappe.assets.clear_local_storage();
-				frappe.assets.set(src, txt);
-			}
-		}
-	},
-
-	get: function (src) {
-		return localStorage.getItem("desk_assets:" + src);
-	},
-
-	set: function (src, txt) {
-		localStorage.setItem("desk_assets:" + src, txt);
-	},
-
-	extn: function (src) {
+	extn(src) {
 		if (src.indexOf("?") != -1) {
 			src = src.split("?").slice(-1)[0];
 		}
 		return src.split(".").slice(-1)[0];
-	},
-
-	handler: {
-		js: function (txt, src) {
-			frappe.dom.eval(txt);
-		},
-		css: function (txt, src) {
-			frappe.dom.set_style(txt);
-		},
-	},
+	}
 
 	bundled_asset(path, is_rtl = null) {
 		if (!path.startsWith("/assets") && path.includes(".bundle.")) {
@@ -197,8 +127,8 @@ frappe.assets = {
 			return path;
 		}
 		return path;
-	},
-};
+	}
+}
 
 function is_reload() {
 	try {
@@ -211,3 +141,5 @@ function is_reload() {
 		return true;
 	}
 }
+
+frappe.assets = new AssetManager();

frappe/public/js/frappe/build_events/build_events.bundle.js

@@ -17,7 +17,7 @@ frappe.realtime.on("build_event", (data) => {
 					if (parts.length === 2) {
 						let filename = parts[0].split("/").slice(-1)[0];
 
-						frappe.assets.executed_ = frappe.assets.executed_.filter(
+						frappe.assets._executed = frappe.assets._executed.filter(
 							(asset) => !asset.includes(`${filename}.bundle`)
 						);
 					}

frappe/public/js/frappe/data_import/import_preview.js

@@ -34,7 +34,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
 					<div class="col-sm-12">
 						<div class="table-actions margin-bottom">
 						</div>
-						<div class="table-preview border"></div>
+						<div class="table-preview"></div>
 						<div class="table-message"></div>
 					</div>
 				</div>

frappe/public/js/frappe/form/controls/code.js

@@ -153,6 +153,7 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex
 			JS: "ace/mode/javascript",
 			Python: "ace/mode/python",
 			Py: "ace/mode/python",
+			PythonExpression: "ace/mode/python",
 			HTML: "ace/mode/html",
 			CSS: "ace/mode/css",
 			Markdown: "ace/mode/markdown",

frappe/public/js/frappe/list/base_list.js

@@ -132,7 +132,9 @@ frappe.views.BaseList = class BaseList {
 			frappe.meta.has_field(doctype, fieldname) ||
 			fieldname === "_seen";
 
-		if (!is_valid_field) {
+		let is_virtual = this.meta.fields.find((df) => df.fieldname == fieldname)?.is_virtual;
+
+		if (!is_valid_field || is_virtual) {
 			return;
 		}
 

frappe/public/js/frappe/list/list_view.js

@@ -374,7 +374,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 					if (frappe.has_indicator(this.doctype) && df.fieldname === "status") {
 						return false;
 					}
-					if (!df.in_list_view) {
+					if (!df.in_list_view || df.is_virtual) {
 						return false;
 					}
 					return df.fieldname !== this.meta.title_field;
@@ -1520,7 +1520,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 	}
 
 	avoid_realtime_update() {
-		if (this.filter_area.is_being_edited()) {
+		if (this.filter_area?.is_being_edited()) {
 			return true;
 		}
 		// this is set when a bulk operation is called from a list view which might update the list view

frappe/public/js/frappe/model/perm.js

@@ -203,7 +203,7 @@ $.extend(frappe.perm, {
 
 		// permission
 		if (p) {
-			if (p.write && !df.disabled) {
+			if (p.write && !df.disabled && !df.is_virtual) {
 				status = "Write";
 			} else if (p.read) {
 				status = "Read";

frappe/public/js/frappe/utils/utils.js

@@ -1705,6 +1705,8 @@ Object.assign(frappe.utils, {
 					fieldname: "source",
 					label: __("Source"),
 					fieldtype: "Data",
+					reqd: 1,
+					description: "The referrer (e.g. google, newsletter)",
 					default: localStorage.getItem("tracker_url:source"),
 				},
 				{
@@ -1719,25 +1721,35 @@ Object.assign(frappe.utils, {
 					fieldname: "medium",
 					label: __("Medium"),
 					fieldtype: "Data",
+					description: "Marketing medium (e.g. cpc, banner, email)",
 					default: localStorage.getItem("tracker_url:medium"),
 				},
+				{
+					fieldname: "content",
+					label: __("Content"),
+					fieldtype: "Data",
+					description: "Use to differentiate ad variants (e.g. A/B testing)",
+					default: localStorage.getItem("tracker_url:content"),
+				},
 			],
 			function (data) {
 				let url = data.url;
 				localStorage.setItem("tracker_url:url", data.url);
 
-				if (data.source) {
-					url += "?source=" + data.source;
-					localStorage.setItem("tracker_url:source", data.source);
-				}
+				url += "?utm_source=" + encodeURIComponent(data.source);
+				localStorage.setItem("tracker_url:source", data.source);
 				if (data.campaign) {
-					url += "&campaign=" + data.campaign;
+					url += "&utm_campaign=" + encodeURIComponent(data.campaign);
 					localStorage.setItem("tracker_url:campaign", data.campaign);
 				}
 				if (data.medium) {
-					url += "&medium=" + data.medium.toLowerCase();
+					url += "&utm_medium=" + encodeURIComponent(data.medium);
 					localStorage.setItem("tracker_url:medium", data.medium);
 				}
+				if (data.medium) {
+					url += "&utm_content=" + encodeURIComponent(data.content);
+					localStorage.setItem("tracker_url:content", data.content);
+				}
 
 				frappe.utils.copy_to_clipboard(url);
 

frappe/public/js/frappe/views/kanban/kanban_board.bundle.js

@@ -377,8 +377,13 @@ frappe.provide("frappe.views");
 		}
 
 		function bind_add_column() {
-			if (!self.board_perms.write) {
+			let doctype = self.cur_list.doctype;
+			let fieldname = self.cur_list.board.field_name;
+			const is_custom_field = frappe.meta.get_docfield(doctype, fieldname)?.is_custom_field;
+
+			if (!self.board_perms.write || !is_custom_field) {
 				// If no write access to board, editing board (by adding column) should be blocked
+				// If standard field then users can't add options
 				self.$kanban_board.find(".add-new-column").remove();
 				return;
 			}

frappe/public/js/onboarding_tours/onboarding_tours.js

@@ -265,7 +265,7 @@ frappe.ui.init_onboarding_tour = () => {
 	typeof frappe.boot.user.onboarding_status == "undefined" &&
 		frappe.boot.user.onboarding_status == {};
 	let route = frappe.router.current_route;
-	if (route[0] === "") return;
+	if (route?.[0] === "") return;
 
 	let tour_name;
 	let matching_tours = [];

frappe/public/scss/desk/data_import.scss

@@ -14,4 +14,16 @@
 
 .table-preview {
 	margin-top: 12px;
+
+	.datatable .dt-scrollable .dt-row:last-child .dt-cell {
+		border-bottom: 1px solid var(--border-color);
+	}
+
+	.dt-row:last-child:not(.dt-row-filter) {
+		border-bottom: none;
+	}
+
+	.datatable .dt-header .dt-row-header {
+		background-color: unset;
+	}
 }

frappe/rate_limiter.py

@@ -138,7 +138,7 @@ def rate_limit(
 			if not identity:
 				frappe.throw(_("Either key or IP flag is required."))
 
-			cache_key = f"rl:{frappe.form_dict.cmd}:{identity}"
+			cache_key = frappe.cache.make_key(f"rl:{frappe.form_dict.cmd}:{identity}")
 
 			value = frappe.cache.get(cache_key)
 			if not value:

frappe/templates/includes/login/login.js

@@ -21,7 +21,8 @@ login.bind_events = function () {
 		args.usr = frappe.utils.xss_sanitise(($("#login_email").val() || "").trim());
 		args.pwd = $("#login_password").val();
 		if (!args.usr || !args.pwd) {
-			frappe.msgprint('{{ _("Both login and password required") }}');
+			{# striptags is used to remove newlines, e is used for escaping #}
+			frappe.msgprint("{{ _('Both login and password required') | striptags | e }}");
 			return false;
 		}
 		login.call(args, null, "/login");
@@ -36,7 +37,7 @@ login.bind_events = function () {
 		args.redirect_to = frappe.utils.sanitise_redirect(frappe.utils.get_url_arg("redirect-to"));
 		args.full_name = frappe.utils.xss_sanitise(($("#signup_fullname").val() || "").trim());
 		if (!args.email || !validate_email(args.email) || !args.full_name) {
-			login.set_status('{{ _("Valid email and name required") }}', 'red');
+			login.set_status({{ _("Valid email and name required") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@@ -49,7 +50,7 @@ login.bind_events = function () {
 		args.cmd = "frappe.core.doctype.user.user.reset_password";
 		args.user = ($("#forgot_email").val() || "").trim();
 		if (!args.user) {
-			login.set_status('{{ _("Valid Login id required.") }}', 'red');
+			login.set_status({{ _("Valid Login id required.") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@@ -62,14 +63,14 @@ login.bind_events = function () {
 		args.cmd = "frappe.www.login.send_login_link";
 		args.email = ($("#login_with_email_link_email").val() || "").trim();
 		if (!args.email) {
-			login.set_status('{{ _("Valid Login id required.") }}', 'red');
+			login.set_status({{ _("Valid Login id required.") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args).then(() => {
-			login.set_status('{{ _("Login link sent to your email") }}', 'blue');
+			login.set_status({{ _("Login link sent to your email") | tojson }}, 'blue');
 			$("#login_with_email_link_email").val("");
 		}).catch(() => {
-			login.set_status('{{ _("Send login link") }}', 'blue');
+			login.set_status({{ _("Send login link") | tojson }}, 'blue');
 		});
 
 		return false;
@@ -79,10 +80,10 @@ login.bind_events = function () {
 		var input = $($(this).attr("toggle"));
 		if (input.attr("type") == "password") {
 			input.attr("type", "text");
-			$(this).text('{{ _("Hide") }}')
+			$(this).text({{ _("Hide") | tojson }})
 		} else {
 			input.attr("type", "password");
-			$(this).text('{{ _("Show") }}')
+			$(this).text({{ _("Show") | tojson }})
 		}
 	});
 
@@ -93,7 +94,7 @@ login.bind_events = function () {
 		args.usr = ($("#login_email").val() || "").trim();
 		args.pwd = $("#login_password").val();
 		if (!args.usr || !args.pwd) {
-			login.set_status('{{ _("Both login and password required") }}', 'red');
+			login.set_status({{ _("Both login and password required") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@@ -168,7 +169,7 @@ login.signup = function () {
 
 // Login
 login.call = function (args, callback, url="/") {
-	login.set_status('{{ _("Verifying...") }}', 'blue');
+	login.set_status({{ _("Verifying...") | tojson }}, 'blue');
 
 	return frappe.call({
 		type: "POST",
@@ -227,13 +228,13 @@ login.login_handlers = (function () {
 	var login_handlers = {
 		200: function (data) {
 			if (data.message == 'Logged In') {
-				login.set_status('{{ _("Success") }}', 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 				document.body.innerHTML = `{% include "templates/includes/splash_screen.html" %}`;
 				window.location.href = frappe.utils.sanitise_redirect(frappe.utils.get_url_arg("redirect-to")) || data.home_page;
 			} else if (data.message == 'Password Reset') {
 				window.location.href = frappe.utils.sanitise_redirect(data.redirect_to);
 			} else if (data.message == "No App") {
-				login.set_status("{{ _('Success') }}", 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 				if (localStorage) {
 					var last_visited =
 						localStorage.getItem("last_visited")
@@ -252,13 +253,13 @@ login.login_handlers = (function () {
 				}
 			} else if (window.location.hash === '#forgot') {
 				if (data.message === 'not found') {
-					login.set_status('{{ _("Not a valid user") }}', 'red');
+					login.set_status({{ _("Not a valid user") | tojson }}, 'red');
 				} else if (data.message == 'not allowed') {
-					login.set_status('{{ _("Not Allowed") }}', 'red');
+					login.set_status({{ _("Not Allowed") | tojson }}, 'red');
 				} else if (data.message == 'disabled') {
-					login.set_status('{{ _("Not Allowed: Disabled User") }}', 'red');
+					login.set_status({{ _("Not Allowed: Disabled User") | tojson }}, 'red');
 				} else {
-					login.set_status('{{ _("Instructions Emailed") }}', 'green');
+					login.set_status({{ _("Instructions Emailed") | tojson }}, 'green');
 				}
 
 
@@ -266,7 +267,7 @@ login.login_handlers = (function () {
 				if (cint(data.message[0]) == 0) {
 					login.set_status(data.message[1], 'red');
 				} else {
-					login.set_status('{{ _("Success") }}', 'green');
+					login.set_status({{ _("Success") | tojson }}, 'green');
 					frappe.msgprint(data.message[1])
 				}
 				//login.set_status(__(data.message), 'green');
@@ -274,7 +275,7 @@ login.login_handlers = (function () {
 
 			//OTP verification
 			if (data.verification && data.message != 'Logged In') {
-				login.set_status('{{ _("Success") }}', 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 
 				document.cookie = "tmp_id=" + data.tmp_id;
 
@@ -287,10 +288,10 @@ login.login_handlers = (function () {
 				}
 			}
 		},
-		401: get_error_handler('{{ _("Invalid Login. Try again.") }}'),
-		417: get_error_handler('{{ _("Oops! Something went wrong.") }}'),
-		404: get_error_handler('{{ _("User does not exist.")}}'),
-		500: get_error_handler('{{ _("Something went wrong.") }}')
+		401: get_error_handler({{ _("Invalid Login. Try again.") | tojson }}),
+		417: get_error_handler({{ _("Oops! Something went wrong.") | tojson }}),
+		404: get_error_handler({{ _("User does not exist.") | tojson }}),
+		500: get_error_handler({{ _("Something went wrong.") | tojson }})
 	};
 
 	return login_handlers;
@@ -322,7 +323,8 @@ var verify_token = function (event) {
 		args.otp = $("#login_token").val();
 		args.tmp_id = frappe.get_cookie('tmp_id');
 		if (!args.otp) {
-			frappe.msgprint('{{ _("Login token required") }}');
+			{# striptags is used to remove newlines, e is used for escaping #}
+			frappe.msgprint("{{ _('Login token required') | striptags | e }}");
 			return false;
 		}
 		login.call(args);
@@ -336,11 +338,11 @@ var request_otp = function (r) {
 		`<div id="twofactor_div">
 			<form class="form-verify">
 				<div class="page-card-head">
-					<span class="indicator blue" data-text="Verification">{{ _("Verification") }}</span>
+					<span class="indicator blue" data-text="Verification">{{ _("Verification") | e }}</span>
 				</div>
 				<div id="otp_div"></div>
-				<input type="text" id="login_token" autocomplete="off" class="form-control" placeholder="{{ _("Verification Code") }}" required="">
-				<button class="btn btn-sm btn-primary btn-block mt-3" id="verify_token">{{ _("Verify") }}</button>
+				<input type="text" id="login_token" autocomplete="off" class="form-control" placeholder="{{ _("Verification Code") | e }}" required="">
+				<button class="btn btn-sm btn-primary btn-block mt-3" id="verify_token">{{ _("Verify") | e }}</button>
 			</form>
 		</div>`
 	);
@@ -354,11 +356,11 @@ var continue_otp_app = function (setup, qrcode) {
 	var qrcode_div = $('<div class="text-muted" style="padding-bottom: 15px;"></div>');
 
 	if (setup) {
-		direction = $('<div>').attr('id', 'qr_info').html('{{ _("Enter Code displayed in OTP App.") }}');
+		direction = $('<div>').attr('id', 'qr_info').text({{ _("Enter Code displayed in OTP App.") | tojson }});
 		qrcode_div.append(direction);
 		$('#otp_div').prepend(qrcode_div);
 	} else {
-		direction = $('<div>').attr('id', 'qr_info').html('{{ _("OTP setup using OTP App was not completed. Please contact Administrator.") }}');
+		direction = $('<div>').attr('id', 'qr_info').text({{ _("OTP setup using OTP App was not completed. Please contact Administrator.") | tojson }});
 		qrcode_div.append(direction);
 		$('#otp_div').prepend(qrcode_div);
 	}
@@ -372,7 +374,7 @@ var continue_sms = function (setup, prompt) {
 		sms_div.append(prompt)
 		$('#otp_div').prepend(sms_div);
 	} else {
-		direction = $('<div>').attr('id', 'qr_info').html(prompt || '{{ _("SMS was not sent. Please contact Administrator.") }}');
+		direction = $('<div>').attr('id', 'qr_info').html(prompt || {{ _("SMS was not sent. Please contact Administrator.") | tojson }});
 		sms_div.append(direction);
 		$('#otp_div').prepend(sms_div)
 	}
@@ -386,7 +388,7 @@ var continue_email = function (setup, prompt) {
 		email_div.append(prompt)
 		$('#otp_div').prepend(email_div);
 	} else {
-		var direction = $('<div>').attr('id', 'qr_info').html(prompt || '{{ _("Verification code email not sent. Please contact Administrator.") }}');
+		var direction = $('<div>').attr('id', 'qr_info').html(prompt || {{ _("Verification code email not sent. Please contact Administrator.") | tojson }});
 		email_div.append(direction);
 		$('#otp_div').prepend(email_div);
 	}

frappe/test_runner.py

@@ -38,6 +38,7 @@ def xmlrunner_wrapper(output):
 
 
 def main(
+	site=None,
 	app=None,
 	module=None,
 	doctype=None,
@@ -50,9 +51,18 @@ def main(
 	doctype_list_path=None,
 	failfast=False,
 	case=None,
+	skip_test_records=False,
+	skip_before_tests=False,
 ):
 	global unittest_runner
 
+	frappe.init(site=site)
+	if not frappe.db:
+		frappe.connect()
+
+	frappe.flags.skip_before_tests = skip_before_tests
+	frappe.flags.skip_test_records = skip_test_records
+
 	if doctype_list_path:
 		app, doctype_list_path = doctype_list_path.split(os.path.sep, 1)
 		with open(frappe.get_app_path(app, doctype_list_path)) as f:
@@ -69,9 +79,6 @@ def main(
 		frappe.flags.print_messages = verbose
 		frappe.flags.in_test = True
 
-		if not frappe.db:
-			frappe.connect()
-
 		# workaround! since there is no separate test db
 		frappe.clear_cache()
 		scheduler_disabled_by_user = frappe.utils.scheduler.is_scheduler_disabled(verbose=False)
@@ -89,9 +96,22 @@ def main(
 				doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output
 			)
 		elif module_def:
-			doctypes = frappe.db.get_list(
-				"DocType", filters={"module": module_def, "istable": 0}, pluck="name"
+			doctypes = []
+			doctypes_ = frappe.get_list(
+				"DocType",
+				filters={"module": module_def, "istable": 0},
+				fields=["name", "module"],
+				as_list=True,
 			)
+			for doctype, module in doctypes_:
+				test_module = get_module_name(doctype, module, "test_", app=app)
+				try:
+					importlib.import_module(test_module)
+				except Exception:
+					pass
+				else:
+					doctypes.append(doctype)
+
 			ret = run_tests_for_doctype(
 				doctypes, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output
 			)
@@ -329,9 +349,6 @@ def _add_test(app, path, filename, verbose, test_suite=None):
 
 
 def make_test_records(doctype, verbose=0, force=False, commit=False):
-	if not frappe.db:
-		frappe.connect()
-
 	if frappe.flags.skip_test_records:
 		return
 

frappe/tests/test_api.py

@@ -15,7 +15,7 @@ from werkzeug.test import TestResponse
 import frappe
 from frappe.installer import update_site_config
 from frappe.tests.utils import FrappeTestCase, patch_hooks
-from frappe.utils import cint, get_test_client, get_url
+from frappe.utils import cint, get_site_url, get_test_client, get_url
 
 try:
 	_site = frappe.local.site
@@ -432,6 +432,21 @@ class TestResponse(FrappeAPITestCase):
 		self.assertGreater(cint(response.headers["content-length"]), 0)
 		self.assertEqual(response.headers["content-disposition"], f'filename="{encoded_filename}"')
 
+	def test_download_private_file_with_unique_url(self):
+		test_content = frappe.generate_hash()
+		file = frappe.get_doc(
+			{
+				"doctype": "File",
+				"file_name": test_content,
+				"content": test_content,
+				"is_private": 1,
+			}
+		)
+		file.insert()
+
+		self.assertEqual(self.get(file.unique_url, {"sid": self.sid}).text, test_content)
+		self.assertEqual(self.get(file.file_url, {"sid": self.sid}).text, test_content)
+
 
 def generate_admin_keys():
 	from frappe.core.doctype.user.user import generate_keys

frappe/tests/test_commands.py

@@ -173,14 +173,18 @@ class BaseTestCommands(FrappeTestCase):
 		cmd_config = {
 			"test_site": TEST_SITE,
 			"admin_password": frappe.conf.admin_password,
-			"root_login": frappe.conf.root_login,
-			"root_password": frappe.conf.root_password,
 			"db_type": frappe.conf.db_type,
+			"db_root_username": frappe.conf.root_login,
+			"db_root_password": frappe.conf.root_password,
 		}
 
 		if not os.path.exists(os.path.join(TEST_SITE, "site_config.json")):
 			cls.execute(
-				"bench new-site {test_site} --admin-password {admin_password} --db-type" " {db_type}",
+				"bench new-site {test_site} "
+				"--admin-password {admin_password} "
+				"--db-root-username {db_root_username} "
+				"--db-root-password {db_root_password} "
+				"--db-type {db_type}",
 				cmd_config,
 			)
 

frappe/tests/test_fmt_money.py

@@ -100,10 +100,3 @@ class TestFmtMoney(FrappeTestCase):
 		frappe.db.set_value("Currency", "JPY", "symbol_on_right", 1)
 		self.assertEqual(fmt_money(100.0, format="#,###.##", currency="JPY"), "100.00 ¥")
 		self.assertEqual(fmt_money(100.0, format="#,###.##", currency="USD"), "$ 100.00")
-
-
-if __name__ == "__main__":
-	import unittest
-
-	frappe.connect()
-	unittest.main()

frappe/translate.py

@@ -20,35 +20,11 @@ from csv import reader, writer
 
 import frappe
 from frappe.gettext.extractors.javascript import extract_javascript
+from frappe.gettext.extractors.utils import extract_messages_from_code, is_translatable
 from frappe.gettext.translate import get_translations_from_mo
-from frappe.model.utils import InvalidIncludePath, render_include
 from frappe.query_builder import DocType, Field
 from frappe.utils import cstr, get_bench_path, is_html, strip, strip_html_tags, unique
 
-TRANSLATE_PATTERN = re.compile(
-	r"_\(\s*"  # starts with literal `_(`, ignore following whitespace/newlines
-	# BEGIN: message search
-	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
-	r"(?P<message>((?!\1).)*)"  # Keep matching until string closing identifier is met which is same as 1st capture group
-	r"\1"  # match exact string closing identifier
-	# END: message search
-	# BEGIN: python context search
-	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
-	r"([\"'])"  # start of context string identifier; 5th capture group
-	r"(?P<py_context>((?!\5).)*)"  # capture context string till closing id is found
-	r"\5"  # match context string closure
-	r")?"  # match 0 or 1 context strings
-	# END: python context search
-	# BEGIN: JS context search
-	r"(\s*,\s*(.)*?\s*(,\s*"  # skip message format replacements: ["format", ...] | null | []
-	r"([\"'])"  # start of context string; 11th capture group
-	r"(?P<js_context>((?!\11).)*)"  # capture context string till closing id is found
-	r"\11"  # match context string closure
-	r")*"
-	r")*"  # match one or more context string
-	# END: JS context search
-	r"\s*\)"  # Closing function call ignore leading whitespace/newlines
-)
 REPORT_TRANSLATE_PATTERN = re.compile('"([^:,^"]*):')
 CSV_STRIP_WHITESPACE_PATTERN = re.compile(r"{\s?([0-9]+)\s?}")
 
@@ -239,9 +215,6 @@ def get_translation_dict_from_file(path, lang, app, throw=False) -> dict[str, st
 
 
 def get_user_translations(lang):
-	if not frappe.db:
-		frappe.connect()
-
 	def _read_from_db():
 		user_translations = {}
 		translations = frappe.get_all(
@@ -676,59 +649,6 @@ def extract_messages_from_javascript_code(code: str) -> list[tuple[int, str, str
 	return messages
 
 
-def extract_messages_from_code(code):
-	"""
-	Extracts translatable strings from a code file
-	:param code: code from which translatable files are to be extracted
-	"""
-	from jinja2 import TemplateError
-
-	try:
-		code = frappe.as_unicode(render_include(code))
-
-	# Exception will occur when it encounters John Resig's microtemplating code
-	except (TemplateError, ImportError, InvalidIncludePath, OSError) as e:
-		if isinstance(e, InvalidIncludePath):
-			frappe.clear_last_message()
-
-	messages = []
-
-	for m in TRANSLATE_PATTERN.finditer(code):
-		message = m.group("message")
-		context = m.group("py_context") or m.group("js_context")
-		pos = m.start()
-
-		if is_translatable(message):
-			messages.append([pos, message, context])
-
-	return add_line_number(messages, code)
-
-
-def is_translatable(m):
-	if (
-		re.search("[a-zA-Z]", m)
-		and not m.startswith("fa fa-")
-		and not m.endswith("px")
-		and not m.startswith("eval:")
-	):
-		return True
-	return False
-
-
-def add_line_number(messages, code):
-	ret = []
-	messages = sorted(messages, key=lambda x: x[0])
-	newlines = [m.start() for m in re.compile(r"\n").finditer(code)]
-	line = 1
-	newline_i = 0
-	for pos, message, context in messages:
-		while newline_i < len(newlines) and pos > newlines[newline_i]:
-			line += 1
-			newline_i += 1
-		ret.append([line, message, context])
-	return ret
-
-
 def read_csv_file(path):
 	"""Read CSV file and return as list of list
 
@@ -1054,9 +974,6 @@ def get_all_languages(with_language_name: bool = False) -> list:
 	def get_all_language_with_name():
 		return frappe.get_all("Language", ["language_code", "language_name"], {"enabled": 1})
 
-	if not frappe.db:
-		frappe.connect()
-
 	if with_language_name:
 		return frappe.cache.get_value("languages_with_name", get_all_language_with_name)
 	else:
@@ -1106,44 +1023,6 @@ def print_language(language: str):
 	frappe.local.jenv = _jenv
 
 
-@functools.total_ordering
-class LazyTranslate:
-	__slots__ = ("msg", "lang", "context")
-
-	def __init__(self, msg: str, lang: str | None = None, context: str | None = None) -> None:
-		self.msg = msg
-		self.lang = lang
-		self.context = context
-
-	@property
-	def value(self) -> str:
-		return frappe._(str(self.msg), self.lang, self.context)
-
-	def __str__(self):
-		return self.value
-
-	def __add__(self, other):
-		if isinstance(other, (str, LazyTranslate)):
-			return self.value + str(other)
-		raise NotImplementedError
-
-	def __radd__(self, other):
-		if isinstance(other, (str, LazyTranslate)):
-			return str(other) + self.value
-		return NotImplementedError
-
-	def __repr__(self) -> str:
-		return f"'{self.value}'"
-
-	# NOTE: it's required to override these methods and raise error as default behaviour will
-	# return `False` in all cases.
-	def __eq__(self, other):
-		raise NotImplementedError
-
-	def __lt__(self, other):
-		raise NotImplementedError
-
-
 # Backward compatibility
 get_full_dict = get_all_translations
 load_lang = get_translations_from_apps

frappe/twofactor.py

@@ -43,11 +43,9 @@ def toggle_two_factor_auth(state, roles=None):
 
 def two_factor_is_enabled(user=None):
 	"""Return True if 2FA is enabled."""
-	enabled = cint(frappe.db.get_single_value("System Settings", "enable_two_factor_auth"))
+	enabled = cint(frappe.get_system_settings("enable_two_factor_auth"))
 	if enabled:
-		bypass_two_factor_auth = cint(
-			frappe.db.get_single_value("System Settings", "bypass_2fa_for_retricted_ip_users")
-		)
+		bypass_two_factor_auth = cint(frappe.get_system_settings("bypass_2fa_for_retricted_ip_users"))
 		if bypass_two_factor_auth and user:
 			user_doc = frappe.get_doc("User", user)
 			restrict_ip_list = (
@@ -144,7 +142,7 @@ def get_otpsecret_for_(user):
 
 
 def get_verification_method():
-	return frappe.db.get_single_value("System Settings", "two_factor_method")
+	return frappe.get_system_settings("two_factor_method")
 
 
 def confirm_otp_token(login_manager, otp=None, tmp_id=None):
@@ -190,7 +188,7 @@ def confirm_otp_token(login_manager, otp=None, tmp_id=None):
 
 
 def get_verification_obj(user, token, otp_secret):
-	otp_issuer = frappe.db.get_single_value("System Settings", "otp_issuer_name")
+	otp_issuer = frappe.get_system_settings("otp_issuer_name")
 	verification_method = get_verification_method()
 	verification_obj = None
 	if verification_method == "SMS":
@@ -263,7 +261,7 @@ def process_2fa_for_email(user, token, otp_secret, otp_issuer, method="Email"):
 def get_email_subject_for_2fa(kwargs_dict):
 	"""Get email subject for 2fa."""
 	subject_template = _("Login Verification Code from {}").format(
-		frappe.db.get_single_value("System Settings", "otp_issuer_name")
+		frappe.get_system_settings("otp_issuer_name")
 	)
 	return frappe.render_template(subject_template, kwargs_dict)
 
@@ -281,7 +279,7 @@ def get_email_body_for_2fa(kwargs_dict):
 def get_email_subject_for_qr_code(kwargs_dict):
 	"""Get QRCode email subject."""
 	subject_template = _("One Time Password (OTP) Registration Code from {}").format(
-		frappe.db.get_single_value("System Settings", "otp_issuer_name")
+		frappe.get_system_settings("otp_issuer_name")
 	)
 	return frappe.render_template(subject_template, kwargs_dict)
 
@@ -299,7 +297,7 @@ def get_link_for_qrcode(user, totp_uri):
 	key = frappe.generate_hash(length=20)
 	key_user = f"{key}_user"
 	key_uri = f"{key}_uri"
-	lifespan = int(frappe.db.get_single_value("System Settings", "lifespan_qrcode_image")) or 240
+	lifespan = int(frappe.get_system_settings("lifespan_qrcode_image")) or 240
 	frappe.cache.set_value(key_uri, totp_uri, expires_in_sec=lifespan)
 	frappe.cache.set_value(key_user, user, expires_in_sec=lifespan)
 	return get_url(f"/qrcode?k={key}")
@@ -433,7 +431,7 @@ def should_remove_barcode_image(barcode):
 	"""Check if it's time to delete barcode image from server."""
 	if isinstance(barcode, str):
 		barcode = frappe.get_doc("File", barcode)
-	lifespan = frappe.db.get_single_value("System Settings", "lifespan_qrcode_image") or 240
+	lifespan = frappe.get_system_settings("lifespan_qrcode_image") or 240
 	if time_diff_in_seconds(get_datetime(), barcode.creation) > int(lifespan):
 		return True
 	return False

frappe/utils/backups.py

@@ -370,6 +370,8 @@ class BackupGenerator:
 			n.write(c.read())
 
 	def take_dump(self):
+		import shlex
+
 		import frappe.utils
 		from frappe.utils.change_log import get_app_branch
 
@@ -419,15 +421,15 @@ class BackupGenerator:
 		extra = []
 		if self.db_type == "mariadb":
 			if self.backup_includes:
-				extra.extend([f"'{x}'" for x in self.backup_includes])
+				extra.extend(self.backup_includes)
 			elif self.backup_excludes:
-				extra.extend([f"--ignore-table='{self.db_name}.{table}'" for table in self.backup_excludes])
+				extra.extend([f"--ignore-table={self.db_name}.{table}" for table in self.backup_excludes])
 
 		elif self.db_type == "postgres":
 			if self.backup_includes:
-				extra.extend([f"--table='public.\"{table}\"'" for table in self.backup_includes])
+				extra.extend([f'--table=public."{table}"' for table in self.backup_includes])
 			elif self.backup_excludes:
-				extra.extend([f"--exclude-table-data='public.\"{table}\"'" for table in self.backup_excludes])
+				extra.extend([f'--exclude-table-data=public."{table}"' for table in self.backup_excludes])
 
 		from frappe.database import get_command
 
@@ -446,11 +448,11 @@ class BackupGenerator:
 				exc=frappe.ExecutableNotFound,
 			)
 		cmd.append(bin)
-		cmd.extend(args)
+		cmd.append(shlex.join(args))
 
 		command = " ".join(["set -o pipefail;"] + cmd + ["|", gzip_exc, ">>", self.backup_path_db])
 		if self.verbose:
-			print(command.replace(frappe.utils.esc(self.password, "$ "), "*" * 10) + "\n")
+			print(command.replace(shlex.quote(self.password), "*" * 10) + "\n")
 
 		frappe.utils.execute_in_shell(command, low_priority=True, check_exit_code=True)
 

frappe/utils/data.py

@@ -2486,18 +2486,27 @@ def is_site_link(link: str) -> bool:
 	return urlparse(link).netloc == urlparse(frappe.utils.get_url()).netloc
 
 
-def add_trackers_to_url(url: str, source: str, campaign: str, medium: str = "email") -> str:
+def add_trackers_to_url(
+	url: str,
+	source: str,
+	campaign: str | None = None,
+	medium: str | None = None,
+	content: str | None = None,
+) -> str:
 	url_parts = list(urlparse(url))
 	if url_parts[0] == "mailto":
 		return url
 
-	trackers = {
-		"source": source,
-		"medium": medium,
-	}
+	trackers = {"utm_source": source}
+
+	if medium:
+		trackers["utm_medium"] = medium
 
 	if campaign:
-		trackers["campaign"] = campaign
+		trackers["utm_campaign"] = campaign
+
+	if content:
+		trackers["utm_content"] = content
 
 	query = dict(parse_qsl(url_parts[4])) | trackers
 

frappe/utils/password.py

@@ -215,4 +215,4 @@ def get_encryption_key():
 
 
 def get_password_reset_limit():
-	return frappe.db.get_single_value("System Settings", "password_reset_limit") or 0
+	return frappe.get_system_settings("password_reset_limit") or 3

frappe/utils/password_strength.py

@@ -50,9 +50,7 @@ default_feedback: "PasswordStrengthFeedback" = {
 def get_feedback(score: int, sequence: list) -> "PasswordStrengthFeedback":
 	"""Return the feedback dictionary consisting of ("warning","suggestions") for the given sequences."""
 	global default_feedback
-	minimum_password_score = int(
-		frappe.db.get_single_value("System Settings", "minimum_password_score") or 2
-	)
+	minimum_password_score = int(frappe.get_system_settings("minimum_password_score") or 2)
 
 	# Starting feedback
 	if len(sequence) == 0:

frappe/utils/pdf.py

@@ -227,7 +227,7 @@ def read_options_from_html(html):
 	return str(soup), options
 
 
-def prepare_header_footer(soup):
+def prepare_header_footer(soup: BeautifulSoup):
 	options = {}
 
 	head = soup.find("head").contents
@@ -238,9 +238,11 @@ def prepare_header_footer(soup):
 
 	# extract header and footer
 	for html_id in ("header-html", "footer-html"):
-		content = soup.find(id=html_id)
-		if content:
-			# there could be multiple instances of header-html/footer-html
+		if content := soup.find(id=html_id):
+			content = content.extract()
+			# `header/footer-html` are extracted, rendered as html
+			# and passed in wkhtmltopdf options (as '--header/footer-html')
+			# Remove instances of them from main content for render_template
 			for tag in soup.find_all(id=html_id):
 				tag.extract()
 

frappe/utils/response.py

@@ -265,7 +265,15 @@ def download_backup(path):
 def download_private_file(path: str) -> Response:
 	"""Checks permissions and sends back private file"""
 
-	files = frappe.get_all("File", filters={"file_url": path}, fields="*")
+	if frappe.session.user == "Guest":
+		raise Forbidden(_("You don't have permission to access this file"))
+
+	filters = {"file_url": path}
+	if frappe.form_dict.fid:
+		filters["name"] = str(frappe.form_dict.fid)
+
+	files = frappe.get_all("File", filters=filters, fields="*")
+
 	# this file might be attached to multiple documents
 	# if the file is accessible from any one of those documents
 	# then it should be downloadable

frappe/website/doctype/blog_settings/blog_settings.json

@@ -1,7 +1,7 @@
 {
  "actions": [],
  "creation": "2013-03-11 17:48:16",
- "description": "Blog Settings",
+ "description": "Settings to control blog categories and interactions like comments and likes",
  "doctype": "DocType",
  "engine": "InnoDB",
  "field_order": [
@@ -131,7 +131,7 @@
  "idx": 1,
  "issingle": 1,
  "links": [],
- "modified": "2022-10-18 15:01:36.202010",
+ "modified": "2024-01-30 14:13:12.477755",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Settings",

frappe/website/doctype/web_page_view/web_page_view.json

@@ -82,6 +82,12 @@
    "fieldtype": "Data",
    "label": "Medium",
    "read_only": 1
+  },
+  {
+   "fieldname": "content",
+   "fieldtype": "Data",
+   "label": "Content",
+   "read_only": 1
   }
  ],
  "in_create": 1,
@@ -111,4 +117,4 @@
  "sort_order": "DESC",
  "states": [],
  "title_field": "path"
-}
+}

frappe/website/doctype/web_page_view/web_page_view.py

@@ -47,6 +47,7 @@ def make_view_log(
 	source=None,
 	campaign=None,
 	medium=None,
+	content=None,
 	visitor_id=None,
 ):
 	if not is_tracking_enabled():
@@ -85,6 +86,7 @@ def make_view_log(
 	view.source = source
 	view.campaign = campaign
 	view.medium = (medium or "").lower()
+	view.content = content
 	view.visitor_id = visitor_id
 
 	try:

frappe/website/report/website_analytics/website_analytics.js

@@ -38,6 +38,7 @@ frappe.query_reports["Website Analytics"] = {
 				{ value: "source", label: __("Source") },
 				{ value: "campaign", label: __("Campaign") },
 				{ value: "medium", label: __("Medium") },
+				{ value: "content", label: __("Content") },
 			],
 			default: "path",
 		},

frappe/website/router.py

@@ -322,7 +322,9 @@ def clear_routing_cache():
 	from frappe.website.doctype.web_form.web_form import get_published_web_forms
 	from frappe.website.doctype.web_page.web_page import get_dynamic_web_pages
 	from frappe.website.page_renderers.document_page import _find_matching_document_webview
+	from frappe.www.sitemap import get_public_pages_from_doctypes
 
 	_find_matching_document_webview.clear_cache()
 	get_dynamic_web_pages.clear_cache()
 	get_published_web_forms.clear_cache()
+	get_public_pages_from_doctypes.clear_cache()

frappe/www/printview.html

@@ -31,14 +31,18 @@
 		document.addEventListener('DOMContentLoaded', () => {
 			const page_div = document.querySelector('.page-break');
 
-			page_div.style.display = 'flex';
-			page_div.style.flexDirection = 'column';
+			if (page_div) {
+				page_div.style.display = 'flex';
+				page_div.style.flexDirection = 'column';
+			}
 
 			const footer_html = document.getElementById('footer-html');
-			footer_html.classList.add('hidden-pdf');
-			footer_html.classList.remove('visible-pdf');
-			footer_html.style.order = 1;
-			footer_html.style.marginTop = '20px';
+			if (footer_html)  {
+				footer_html.classList.add('hidden-pdf');
+				footer_html.classList.remove('visible-pdf');
+				footer_html.style.order = 1;
+				footer_html.style.marginTop = '20px';
+			}
 		});
 	</script>
 </body>

frappe/www/printview.py

@@ -198,11 +198,23 @@ def get_rendered_template(
 		letter_head.content = frappe.utils.jinja.render_template(
 			letter_head.content, {"doc": doc.as_dict()}
 		)
+		if letter_head.header_script:
+			letter_head.content += f"""
+				<script>
+					{ letter_head.header_script }
+				</script>
+			"""
 
 	if letter_head.footer:
 		letter_head.footer = frappe.utils.jinja.render_template(
 			letter_head.footer, {"doc": doc.as_dict()}
 		)
+		if letter_head.footer_script:
+			letter_head.footer += f"""
+				<script>
+					{ letter_head.footer_script }
+				</script>
+			"""
 
 	convert_markdown(doc)
 
@@ -391,13 +403,24 @@ def validate_key(key: str, doc: "Document") -> None:
 def get_letter_head(doc: "Document", no_letterhead: bool, letterhead: str | None = None) -> dict:
 	if no_letterhead:
 		return {}
-	if letterhead:
-		return frappe.db.get_value("Letter Head", letterhead, ["content", "footer"], as_dict=True)
-	if doc.get("letter_head"):
-		return frappe.db.get_value("Letter Head", doc.letter_head, ["content", "footer"], as_dict=True)
+
+	letterhead_name = letterhead or doc.get("letter_head")
+	if letterhead_name:
+		return frappe.db.get_value(
+			"Letter Head",
+			letterhead_name,
+			["content", "footer", "header_script", "footer_script"],
+			as_dict=True,
+		)
 	else:
 		return (
-			frappe.db.get_value("Letter Head", {"is_default": 1}, ["content", "footer"], as_dict=True) or {}
+			frappe.db.get_value(
+				"Letter Head",
+				{"is_default": 1},
+				["content", "footer", "header_script", "footer_script"],
+				as_dict=True,
+			)
+			or {}
 		)
 
 

frappe/www/sitemap.py

@@ -6,6 +6,7 @@ from urllib.parse import quote
 import frappe
 from frappe.model.document import get_controller
 from frappe.utils import get_url, nowdate
+from frappe.utils.caching import redis_cache
 from frappe.website.router import get_pages
 
 no_cache = 1
@@ -31,42 +32,40 @@ def get_context(context):
 	return {"links": links}
 
 
+@redis_cache(ttl=6 * 60 * 60)
 def get_public_pages_from_doctypes():
 	"""Return pages from doctypes that are publicly accessible."""
 
-	def get_sitemap_routes():
-		routes = {}
-		doctypes_with_web_view = frappe.get_all(
-			"DocType",
-			filters={"has_web_view": True, "allow_guest_to_view": True},
-			pluck="name",
-		)
+	routes = {}
+	doctypes_with_web_view = frappe.get_all(
+		"DocType",
+		filters={"has_web_view": True, "allow_guest_to_view": True},
+		pluck="name",
+	)
 
-		for doctype in doctypes_with_web_view:
-			controller = get_controller(doctype)
-			meta = frappe.get_meta(doctype)
-			condition_field = meta.is_published_field or controller.website.condition_field
+	for doctype in doctypes_with_web_view:
+		controller = get_controller(doctype)
+		meta = frappe.get_meta(doctype)
+		condition_field = meta.is_published_field or controller.website.condition_field
 
-			if not condition_field:
-				continue
+		if not condition_field:
+			continue
 
-			try:
-				res = frappe.get_all(
-					doctype,
-					fields=["route", "name", "modified"],
-					filters={condition_field: True},
-				)
-			except Exception as e:
-				if not frappe.db.is_missing_column(e):
-					raise e
+		try:
+			res = frappe.get_all(
+				doctype,
+				fields=["route", "name", "modified"],
+				filters={condition_field: True},
+			)
+		except Exception as e:
+			if not frappe.db.is_missing_column(e):
+				raise e
 
-			for r in res:
-				routes[r.route] = {
-					"doctype": doctype,
-					"name": r.name,
-					"modified": r.modified,
-				}
+		for r in res:
+			routes[r.route] = {
+				"doctype": doctype,
+				"name": r.name,
+				"modified": r.modified,
+			}
 
-		return routes
-
-	return frappe.cache.get_value("sitemap_routes", get_sitemap_routes)
+	return routes

frappe/www/website_script.js

@@ -32,12 +32,13 @@ ga('send', 'pageview');
 						browser: browser.name,
 						version: browser.version,
 						user_tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
-						source: query_params.source,
-						medium: query_params.medium,
-						campaign: query_params.campaign,
+						source: query_params.source || query_params.utm_source,
+						medium: query_params.medium || query_params.utm_medium,
+						campaign: query_params.campaign || query_params.utm_campaign,
+						content: query_params.content || query_params.utm_content,
 						visitor_id: result.visitorId
 					})
 			})
 		})
 	}
-{% endif %}
+{% endif %}

pyproject.toml

@@ -31,7 +31,6 @@ dependencies = [
     "beautifulsoup4~=4.12.2",
     "bleach-allowlist~=1.0.3",
     "bleach[css]~=6.0.0",
-    "cairocffi==1.5.1",
     "chardet~=5.1.0",
     "croniter~=2.0.1",
     "cryptography~=41.0.3",