Merge remote-tracking branch 'upstream/develop' into copy-config-to-new-app

cypress/integration/control_currency.js

@@ -9,6 +9,7 @@ context("Control Currency", () => {
 	function get_dialog_with_currency(df_options = {}) {
 		return cy.dialog({
 			title: "Currency Check",
+			animate: false,
 			fields: [
 				{
 					fieldname: fieldname,
@@ -76,6 +77,7 @@ context("Control Currency", () => {
 				});
 
 			get_dialog_with_currency(test_case.df_options).as("dialog");
+			cy.wait(300);
 			cy.get_field(fieldname, "Currency").clear();
 			cy.wait(300);
 			cy.fill_field(fieldname, test_case.input, "Currency").blur();

cypress/integration/control_float.js

@@ -7,6 +7,7 @@ context("Control Float", () => {
 	function get_dialog_with_float() {
 		return cy.dialog({
 			title: "Float Check",
+			animate: false,
 			fields: [
 				{
 					fieldname: "float_number",
@@ -19,6 +20,7 @@ context("Control Float", () => {
 
 	it("check value changes", () => {
 		get_dialog_with_float().as("dialog");
+		cy.wait(300);
 
 		let data = get_data();
 		data.forEach((x) => {

frappe/__init__.py

@@ -109,7 +109,7 @@ class _dict(dict):
 
 
 def _(msg: str, lang: str | None = None, context: str | None = None) -> str:
-	"""Returns translated string in current lang, if exists.
+	"""Return translated string in current lang, if exists.
 	Usage:
 	        _('Change')
 	        _('Change', context='Coins')
@@ -144,8 +144,8 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str:
 	return translated_string or non_translated_string
 
 
-def as_unicode(text: str, encoding: str = "utf-8") -> str:
-	"""Convert to unicode if required"""
+def as_unicode(text, encoding: str = "utf-8") -> str:
+	"""Convert to unicode if required."""
 	if isinstance(text, str):
 		return text
 	elif text is None:
@@ -327,7 +327,7 @@ def connect_replica() -> bool:
 
 
 def get_site_config(sites_path: str | None = None, site_path: str | None = None) -> dict[str, Any]:
-	"""Returns `site_config.json` combined with `sites/common_site_config.json`.
+	"""Return `site_config.json` combined with `sites/common_site_config.json`.
 	`site_config` is a set of site wide settings like database name, password, email etc."""
 	config = _dict()
 
@@ -373,7 +373,7 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 
 
 def get_common_site_config(sites_path: str | None = None) -> dict[str, Any]:
-	"""Returns common site config as dictionary.
+	"""Return common site config as dictionary.
 
 	This is useful for:
 	- checking configuration which should only be allowed in common site config
@@ -432,7 +432,7 @@ def setup_redis_cache_connection():
 
 
 def get_traceback(with_context: bool = False) -> str:
-	"""Returns error traceback."""
+	"""Return error traceback."""
 	from frappe.utils import get_traceback
 
 	return get_traceback(with_context=with_context)
@@ -638,7 +638,7 @@ def get_user():
 
 
 def get_roles(username=None) -> list[str]:
-	"""Returns roles of current user."""
+	"""Return roles of current user."""
 	if not local.session or not local.session.user:
 		return ["Guest"]
 	import frappe.permissions
@@ -780,9 +780,9 @@ def sendmail(
 	return builder.process(send_now=now)
 
 
-whitelisted = []
-guest_methods = []
-xss_safe_methods = []
+whitelisted = set()
+guest_methods = set()
+xss_safe_methods = set()
 allowed_http_methods_for_whitelisted_func = {}
 
 
@@ -821,14 +821,14 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None):
 		else:
 			fn = validate_argument_types(fn, apply_condition=in_request_or_test)
 
-		whitelisted.append(fn)
+		whitelisted.add(fn)
 		allowed_http_methods_for_whitelisted_func[fn] = methods
 
 		if allow_guest:
-			guest_methods.append(fn)
+			guest_methods.add(fn)
 
 			if xss_safe:
-				xss_safe_methods.append(fn)
+				xss_safe_methods.add(fn)
 
 		return method or fn
 
@@ -1003,8 +1003,9 @@ def has_permission(
 	parent_doctype=None,
 ):
 	"""
-	Returns True if the user has permission `ptype` for given `doctype` or `doc`
-	Raises `frappe.PermissionError` if user isn't permitted and `throw` is truthy
+	Return True if the user has permission `ptype` for given `doctype` or `doc`.
+
+	Raise `frappe.PermissionError` if user isn't permitted and `throw` is truthy
 
 	:param doctype: DocType for which permission is to be check.
 	:param ptype: Permission type (`read`, `write`, `create`, `submit`, `cancel`, `amend`). Default: `read`.
@@ -1084,7 +1085,7 @@ def has_website_permission(doc=None, ptype="read", user=None, verbose=False, doc
 
 
 def is_table(doctype: str) -> bool:
-	"""Returns True if `istable` property (indicating child Table) is set for given DocType."""
+	"""Return True if `istable` property (indicating child Table) is set for given DocType."""
 
 	def get_tables():
 		return db.get_values("DocType", filters={"istable": 1}, order_by=None, pluck=True)
@@ -1128,7 +1129,7 @@ def new_doc(
 	as_dict: bool = False,
 	**kwargs,
 ) -> "Document":
-	"""Returns a new document of the given DocType with defaults set.
+	"""Return a new document of the given DocType with defaults set.
 
 	:param doctype: DocType of the new document.
 	:param parent_doc: [optional] add to parent document.
@@ -1152,6 +1153,7 @@ def set_value(doctype, docname, fieldname, value=None):
 
 
 def get_cached_doc(*args, **kwargs) -> "Document":
+	"""Identical to `frappe.get_doc`, but return from cache if available."""
 	if (key := can_cache_doc(args)) and (doc := cache.get_value(key)):
 		return doc
 
@@ -1174,7 +1176,7 @@ def _set_document_in_cache(key: str, doc: "Document") -> None:
 def can_cache_doc(args) -> str | None:
 	"""
 	Determine if document should be cached based on get_doc params.
-	Returns cache key if doc can be cached, None otherwise.
+	Return cache key if doc can be cached, None otherwise.
 	"""
 
 	if not args:
@@ -1426,17 +1428,17 @@ def rename_doc(
 
 
 def get_module(modulename):
-	"""Returns a module object for given Python module name using `importlib.import_module`."""
+	"""Return a module object for given Python module name using `importlib.import_module`."""
 	return importlib.import_module(modulename)
 
 
 def scrub(txt: str) -> str:
-	"""Returns sluggified string. e.g. `Sales Order` becomes `sales_order`."""
+	"""Return sluggified string. e.g. `Sales Order` becomes `sales_order`."""
 	return cstr(txt).replace(" ", "_").replace("-", "_").lower()
 
 
 def unscrub(txt: str) -> str:
-	"""Returns titlified string. e.g. `sales_order` becomes `Sales Order`."""
+	"""Return titlified string. e.g. `sales_order` becomes `Sales Order`."""
 	return txt.replace("_", " ").replace("-", " ").title()
 
 
@@ -1536,7 +1538,7 @@ def get_installed_apps(*, _ensure_on_bench=False) -> list[str]:
 
 
 def get_doc_hooks():
-	"""Returns hooked methods for given doc. It will expand the dict tuple if required."""
+	"""Return hooked methods for given doc. Expand the dict tuple if required."""
 	if not hasattr(local, "doc_events_hooks"):
 		hooks = get_hooks("doc_events", {})
 		out = {}
@@ -1643,7 +1645,7 @@ def setup_module_map():
 
 
 def get_file_items(path, raise_not_found=False, ignore_empty_lines=True):
-	"""Returns items from text file as a list. Ignores empty lines."""
+	"""Return items from text file as a list. Ignore empty lines."""
 	import frappe.utils
 
 	content = read_file(path, raise_not_found=raise_not_found)
@@ -1996,7 +1998,7 @@ def get_all(doctype, *args, **kwargs):
 
 
 def get_value(*args, **kwargs):
-	"""Returns a document property or list of properties.
+	"""Return a document property or list of properties.
 
 	Alias for `frappe.db.get_value`
 
@@ -2011,6 +2013,7 @@ def get_value(*args, **kwargs):
 
 
 def as_json(obj: dict | list, indent=1, separators=None, ensure_ascii=True) -> str:
+	"""Return the JSON string representation of the given `obj`."""
 	from frappe.utils.response import json_handler
 
 	if separators is None:
@@ -2043,7 +2046,7 @@ def are_emails_muted():
 
 
 def get_test_records(doctype):
-	"""Returns list of objects from `test_records.json` in the given doctype's folder."""
+	"""Return list of objects from `test_records.json` in the given doctype's folder."""
 	from frappe.modules import get_doctype_module, get_module_path
 
 	path = os.path.join(
@@ -2276,7 +2279,7 @@ log_level = None
 def logger(
 	module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20
 ):
-	"""Returns a python logger that uses StreamHandler"""
+	"""Return a python logger that uses StreamHandler."""
 	from frappe.utils.logger import get_logger
 
 	return get_logger(
@@ -2296,7 +2299,8 @@ def get_desk_link(doctype, name):
 	return html.format(doctype=doctype, name=name, doctype_local=_(doctype))
 
 
-def bold(text):
+def bold(text: str) -> str:
+	"""Return `text` wrapped in `<strong>` tags."""
 	return f"<strong>{text}</strong>"
 
 
@@ -2319,7 +2323,8 @@ def get_website_settings(key):
 	return local.website_settings.get(key)
 
 
-def get_system_settings(key):
+def get_system_settings(key: str):
+	"""Return the value associated with the given `key` from System Settings DocType."""
 	if not hasattr(local, "system_settings"):
 		try:
 			local.system_settings = get_cached_doc("System Settings")
@@ -2338,7 +2343,7 @@ def get_active_domains():
 
 def get_version(doctype, name, limit=None, head=False, raise_err=True):
 	"""
-	Returns a list of version information of a given DocType.
+	Return a list of version information for the given DocType.
 
 	Note: Applicable only if DocType has changes tracked.
 

frappe/auth.py

@@ -289,7 +289,7 @@ class LoginManager:
 	def check_password(self, user, pwd):
 		"""check password"""
 		try:
-			# returns user in correct case
+			# return user in correct case
 			return check_password(user, pwd)
 		except frappe.AuthenticationError:
 			self.fail("Incorrect password", user=user)

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -297,11 +297,11 @@ class AutoRepeat(Document):
 
 	def get_next_schedule_date(self, schedule_date, for_full_schedule=False):
 		"""
-		Returns the next schedule date for auto repeat after a recurring document has been created.
-		Adds required offset to the schedule_date param and returns the next schedule date.
+		Return the next schedule date for auto repeat after a recurring document has been created.
+		Add required offset to the schedule_date param and return the next schedule date.
 
 		:param schedule_date: The date when the last recurring document was created.
-		:param for_full_schedule: If True, returns the immediate next schedule date, else the full schedule.
+		:param for_full_schedule: If True, return the immediate next schedule date, else the full schedule.
 		"""
 		if month_map.get(self.frequency):
 			month_count = month_map.get(self.frequency) + month_diff(schedule_date, self.start_date) - 1

frappe/build.py

@@ -133,10 +133,9 @@ def setup_assets(assets_archive):
 	return directories_created
 
 
-def download_frappe_assets(verbose=True):
-	"""Downloads and sets up Frappe assets if they exist based on the current
-	commit HEAD.
-	Returns True if correctly setup else returns False.
+def download_frappe_assets(verbose=True) -> bool:
+	"""Download and set up Frappe assets if they exist based on the current commit HEAD.
+	Return True if correctly setup else return False.
 	"""
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
 
@@ -407,7 +406,7 @@ def link_assets_dir(source, target, hard_link=False):
 
 
 def scrub_html_template(content):
-	"""Returns HTML content with removed whitespace and comments"""
+	"""Return HTML content with removed whitespace and comments."""
 	# remove whitespace to a single space
 	content = WHITESPACE_PATTERN.sub(" ", content)
 
@@ -418,7 +417,7 @@ def scrub_html_template(content):
 
 
 def html_to_js_template(path, content):
-	"""returns HTML template content as Javascript code, adding it to `frappe.templates`"""
+	"""Return HTML template content as Javascript code, by adding it to `frappe.templates`."""
 	return """frappe.templates["{key}"] = '{content}';\n""".format(
 		key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content)
 	)

frappe/client.py

@@ -38,7 +38,7 @@ def get_list(
 	as_dict: bool = True,
 	or_filters=None,
 ):
-	"""Returns a list of records by filters, fields, ordering and limit
+	"""Return a list of records by filters, fields, ordering and limit.
 
 	:param doctype: DocType of the data to be queried
 	:param fields: fields to be returned. Default is `name`
@@ -74,7 +74,7 @@ def get_count(doctype, filters=None, debug=False, cache=False):
 
 @frappe.whitelist()
 def get(doctype, name=None, filters=None, parent=None):
-	"""Returns a document by name or filters
+	"""Return a document by name or filters.
 
 	:param doctype: DocType of the document to be returned
 	:param name: return document of this `name`
@@ -97,7 +97,7 @@ def get(doctype, name=None, filters=None, parent=None):
 
 @frappe.whitelist()
 def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, parent=None):
-	"""Returns a value form a document
+	"""Return a value from a document.
 
 	:param doctype: DocType to be queried
 	:param fieldname: Field to be returned (default `name`)
@@ -296,7 +296,7 @@ def bulk_update(docs):
 
 @frappe.whitelist()
 def has_permission(doctype, docname, perm_type="read"):
-	"""Returns a JSON with data whether the document has the requested permission
+	"""Return a JSON with data whether the document has the requested permission.
 
 	:param doctype: DocType of the document to be checked
 	:param docname: `name` of the document to be checked
@@ -307,7 +307,7 @@ def has_permission(doctype, docname, perm_type="read"):
 
 @frappe.whitelist()
 def get_doc_permissions(doctype, docname):
-	"""Returns an evaluated document permissions dict like `{"read":1, "write":1}`
+	"""Return an evaluated document permissions dict like `{"read":1, "write":1}`.
 
 	:param doctype: DocType of the document to be evaluated
 	:param docname: `name` of the document to be evaluated
@@ -354,7 +354,7 @@ def get_js(items):
 
 @frappe.whitelist(allow_guest=True)
 def get_time_zone():
-	"""Returns default time zone"""
+	"""Return the default time zone."""
 	return {"time_zone": frappe.defaults.get_defaults().get("time_zone")}
 
 
@@ -466,8 +466,7 @@ def validate_link(doctype: str, docname: str, fields=None):
 
 
 def insert_doc(doc) -> "Document":
-	"""Inserts document and returns parent document object with appended child document
-	if `doc` is child document else returns the inserted document object
+	"""Insert document and return parent document object with appended child document if `doc` is child document else return the inserted document object.
 
 	:param doc: doc to insert (dict)"""
 

frappe/commands/site.py

@@ -72,13 +72,10 @@ def new_site(
 	setup_db=True,
 ):
 	"Create a new site"
-	from frappe.installer import _new_site, extract_sql_from_archive
+	from frappe.installer import _new_site
 
 	frappe.init(site=site, new_site=True)
 
-	if source_sql:
-		source_sql = extract_sql_from_archive(source_sql)
-
 	_new_site(
 		db_name,
 		site,
@@ -180,75 +177,113 @@ def _restore(
 	with_public_files=None,
 	with_private_files=None,
 ):
+	from frappe.installer import extract_files
+	from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key
 
-	from frappe.installer import (
-		_new_site,
-		extract_files,
-		extract_sql_from_archive,
-		is_downgrade,
-		is_partial,
-		validate_database_sql,
-	)
-	from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key
+	err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True)
+	if err:
+		click.secho("Failed to detect type of backup file", fg="red")
+		sys.exit(1)
 
-	_backup = Backup(sql_file_path)
-
-	try:
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
-		if is_partial(decompressed_file_name):
-			click.secho(
-				"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
-				fg="red",
-			)
-			click.secho(
-				"Use `bench partial-restore` to restore a partial backup to an existing site.",
-				fg="yellow",
-			)
-			_backup.decryption_rollback()
-			sys.exit(1)
-
-	except UnicodeDecodeError:
-		_backup.decryption_rollback()
+	if "cipher" in out.decode().split(":")[-1].strip():
 		if encryption_key:
 			click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow")
-			_backup.backup_decryption(encryption_key)
 
 		else:
 			click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow")
 			encryption_key = get_or_generate_backup_encryption_key()
-			_backup.backup_decryption(encryption_key)
 
-		# Rollback on unsuccessful decryrption
-		if not os.path.exists(sql_file_path):
-			click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
+		with decrypt_backup(sql_file_path, encryption_key):
+			# Rollback on unsuccessful decryption
+			if not os.path.exists(sql_file_path):
+				click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
+				sys.exit(1)
 
-			_backup.decryption_rollback()
-			sys.exit(1)
-
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
-
-		if is_partial(decompressed_file_name):
-			click.secho(
-				"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
-				fg="red",
+			restore_backup(
+				sql_file_path,
+				site,
+				db_root_username,
+				db_root_password,
+				verbose,
+				install_app,
+				admin_password,
+				force,
 			)
-			click.secho(
-				"Use `bench partial-restore` to restore a partial backup to an existing site.",
-				fg="yellow",
-			)
-			_backup.decryption_rollback()
-			sys.exit(1)
+	else:
+		restore_backup(
+			sql_file_path,
+			site,
+			db_root_username,
+			db_root_password,
+			verbose,
+			install_app,
+			admin_password,
+			force,
+		)
 
-	validate_database_sql(decompressed_file_name, _raise=not force)
+	# Extract public and/or private files to the restored site, if user has given the path
+	if with_public_files:
+		# Decrypt data if there is a Key
+		if encryption_key:
+			with decrypt_backup(with_public_files, encryption_key):
+				public = extract_files(site, with_public_files)
+		else:
+			public = extract_files(site, with_public_files)
 
-	# dont allow downgrading to older versions of frappe without force
-	if not force and is_downgrade(decompressed_file_name, verbose=True):
+		# Removing temporarily created file
+		os.remove(public)
+
+	if with_private_files:
+		# Decrypt data if there is a Key
+		if encryption_key:
+			with decrypt_backup(with_private_files, encryption_key):
+				private = extract_files(site, with_private_files)
+		else:
+			private = extract_files(site, with_private_files)
+
+		# Removing temporarily created file
+		os.remove(private)
+
+	success_message = "Site {} has been restored{}".format(
+		site, " with files" if (with_public_files or with_private_files) else ""
+	)
+	click.secho(success_message, fg="green")
+
+
+def restore_backup(
+	sql_file_path: str,
+	site,
+	db_root_username,
+	db_root_password,
+	verbose,
+	install_app,
+	admin_password,
+	force,
+):
+	from frappe.installer import _new_site, is_downgrade, is_partial, validate_database_sql
+
+	if is_partial(sql_file_path):
+		click.secho(
+			"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
+			fg="red",
+		)
+		click.secho(
+			"Use `bench partial-restore` to restore a partial backup to an existing site.",
+			fg="yellow",
+		)
+		sys.exit(1)
+
+	# Check if the backup is of an older version of frappe and the user hasn't specified force
+	if is_downgrade(sql_file_path, verbose=True) and not force:
 		warn_message = (
 			"This is not recommended and may lead to unexpected behaviour. "
 			"Do you want to continue anyway?"
 		)
 		click.confirm(warn_message, abort=True)
 
+	# Validate the sql file
+	validate_database_sql(sql_file_path, _raise=not force)
+
 	try:
 		_new_site(
 			frappe.conf.db_name,
@@ -258,53 +293,15 @@ def _restore(
 			admin_password=admin_password,
 			verbose=verbose,
 			install_apps=install_app,
-			source_sql=decompressed_file_name,
+			source_sql=sql_file_path,
 			force=True,
 			db_type=frappe.conf.db_type,
 		)
 
 	except Exception as err:
 		print(err.args[1])
-		_backup.decryption_rollback()
 		sys.exit(1)
 
-	# Removing temporarily created file
-	if decompressed_file_name != sql_file_path:
-		os.remove(decompressed_file_name)
-		_backup.decryption_rollback()
-
-	# Extract public and/or private files to the restored site, if user has given the path
-	if with_public_files:
-		# Decrypt data if there is a Key
-		if encryption_key:
-			_backup = Backup(with_public_files)
-			_backup.backup_decryption(encryption_key)
-			if not os.path.exists(with_public_files):
-				_backup.decryption_rollback()
-		public = extract_files(site, with_public_files)
-
-		# Removing temporarily created file
-		os.remove(public)
-		_backup.decryption_rollback()
-
-	if with_private_files:
-		# Decrypt data if there is a Key
-		if encryption_key:
-			_backup = Backup(with_private_files)
-			_backup.backup_decryption(encryption_key)
-			if not os.path.exists(with_private_files):
-				_backup.decryption_rollback()
-		private = extract_files(site, with_private_files)
-
-		# Removing temporarily created file
-		os.remove(private)
-		_backup.decryption_rollback()
-
-	success_message = "Site {} has been restored{}".format(
-		site, " with files" if (with_public_files or with_private_files) else ""
-	)
-	click.secho(success_message, fg="green")
-
 
 @click.command("partial-restore")
 @click.argument("sql-file-path")
@@ -312,38 +309,23 @@ def _restore(
 @click.option("--encryption-key", help="Backup encryption key")
 @pass_context
 def partial_restore(context, sql_file_path, verbose, encryption_key=None):
-	from frappe.installer import extract_sql_from_archive, partial_restore
-	from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key
+	from frappe.installer import is_partial, partial_restore
+	from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key
 
 	if not os.path.exists(sql_file_path):
 		print("Invalid path", sql_file_path)
 		sys.exit(1)
 
 	site = get_site(context)
-	frappe.init(site=site)
-
-	_backup = Backup(sql_file_path)
-
 	verbose = context.verbose or verbose
-
+	frappe.init(site=site)
 	frappe.connect(site=site)
-	try:
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
+	err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True)
+	if err:
+		click.secho("Failed to detect type of backup file", fg="red")
+		sys.exit(1)
 
-		with open(decompressed_file_name) as f:
-			header = " ".join(f.readline() for _ in range(5))
-
-			# Check for full backup file
-			if "Partial Backup" not in header:
-				click.secho(
-					"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
-					fg="red",
-				)
-				_backup.decryption_rollback()
-				sys.exit(1)
-
-	except UnicodeDecodeError:
-		_backup.decryption_rollback()
+	if "cipher" in out.decode().split(":")[-1].strip():
 		if encryption_key:
 			click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow")
 			key = encryption_key
@@ -352,35 +334,30 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None):
 			click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow")
 			key = get_or_generate_backup_encryption_key()
 
-		_backup.backup_decryption(key)
-
-		# Rollback on unsuccessful decryrption
-		if not os.path.exists(sql_file_path):
-			click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
-			_backup.decryption_rollback()
-			sys.exit(1)
-
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
-
-		with open(decompressed_file_name) as f:
-			header = " ".join(f.readline() for _ in range(5))
-
-			# Check for Full backup file.
-			if "Partial Backup" not in header:
+		with decrypt_backup(sql_file_path, key):
+			if not is_partial(sql_file_path):
 				click.secho(
-					"Full Backup file detected.Use `bench restore` to restore a Frappe Site.",
+					"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
 					fg="red",
 				)
-				_backup.decryption_rollback()
 				sys.exit(1)
 
-	partial_restore(sql_file_path, verbose)
+			partial_restore(sql_file_path, verbose)
 
-	# Removing temporarily created file
-	_backup.decryption_rollback()
-	if os.path.exists(sql_file_path.rstrip(".gz")):
-		os.remove(sql_file_path.rstrip(".gz"))
+		# Rollback on unsuccessful decryption
+		if not os.path.exists(sql_file_path):
+			click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
+			sys.exit(1)
 
+	else:
+		if not is_partial(sql_file_path):
+			click.secho(
+				"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
+				fg="red",
+			)
+			sys.exit(1)
+
+		partial_restore(sql_file_path, verbose)
 	frappe.destroy()
 
 
@@ -865,6 +842,9 @@ def use(site, sites_path="."):
 )
 @click.option("--verbose", default=False, is_flag=True, help="Add verbosity")
 @click.option("--compress", default=False, is_flag=True, help="Compress private and public files")
+@click.option(
+	"--old-backup-metadata", default=False, is_flag=True, help="Use older backup metadata"
+)
 @pass_context
 def backup(
 	context,
@@ -879,6 +859,7 @@ def backup(
 	compress=False,
 	include="",
 	exclude="",
+	old_backup_metadata=False,
 ):
 	"Backup"
 
@@ -904,6 +885,7 @@ def backup(
 				compress=compress,
 				verbose=verbose,
 				force=True,
+				old_backup_metadata=old_backup_metadata,
 			)
 		except Exception:
 			click.secho(

frappe/contacts/doctype/address/address.py

@@ -143,7 +143,7 @@ def get_preferred_address(doctype, name, preferred_key="is_primary_address"):
 def get_default_address(
 	doctype: str, name: str | None, sort_key: str = "is_primary_address"
 ) -> str | None:
-	"""Returns default Address name for the given doctype, name"""
+	"""Return default Address name for the given doctype, name."""
 	if sort_key not in ["is_shipping_address", "is_primary_address"]:
 		return None
 
@@ -228,7 +228,7 @@ def get_address_list(doctype, txt, filters, limit_start, limit_page_length=20, o
 
 
 def has_website_permission(doc, ptype, user, verbose=False):
-	"""Returns true if there is a related lead or contact related to this document"""
+	"""Return True if there is a related lead or contact related to this document."""
 	contact_name = frappe.db.get_value("Contact", {"email_id": frappe.session.user})
 
 	if contact_name:

frappe/contacts/doctype/contact/contact.py

@@ -168,7 +168,7 @@ class Contact(Document):
 
 
 def get_default_contact(doctype, name):
-	"""Returns default contact for the given doctype, name"""
+	"""Return default contact for the given doctype, name."""
 	out = frappe.db.sql(
 		"""select parent,
 			IFNULL((select is_primary_contact from tabContact c where c.name = dl.parent), 0)

frappe/core/api/file.py

@@ -15,8 +15,7 @@ def unzip_file(name: str):
 
 @frappe.whitelist()
 def get_attached_images(doctype: str, names: list[str] | str) -> frappe._dict:
-	"""get list of image urls attached in form
-	returns {name: ['image.jpg', 'image.png']}"""
+	"""Return list of image urls attached in form `{name: ['image.jpg', 'image.png']}`."""
 
 	if isinstance(names, str):
 		names = json.loads(names)

frappe/core/doctype/communication/communication.py

@@ -298,7 +298,7 @@ class Communication(Document, CommunicationEmailMixin):
 
 	@staticmethod
 	def _get_emails_list(emails=None, exclude_displayname=False):
-		"""Returns list of emails from given email string.
+		"""Return list of emails from given email string.
 
 		* Removes duplicate mailids
 		* Removes display name from email address if exclude_displayname is True
@@ -309,15 +309,15 @@ class Communication(Document, CommunicationEmailMixin):
 		return [email.lower() for email in set(emails) if email]
 
 	def to_list(self, exclude_displayname=True):
-		"""Returns to list."""
+		"""Return `to` list."""
 		return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname)
 
 	def cc_list(self, exclude_displayname=True):
-		"""Returns cc list."""
+		"""Return `cc` list."""
 		return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname)
 
 	def bcc_list(self, exclude_displayname=True):
-		"""Returns bcc list."""
+		"""Return `bcc` list."""
 		return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname)
 
 	def get_attachments(self):
@@ -438,7 +438,7 @@ class Communication(Document, CommunicationEmailMixin):
 				frappe.db.commit()
 
 	def parse_email_for_timeline_links(self):
-		if not frappe.db.get_value("Email Account", self.email_account, "enable_automatic_linking"):
+		if not frappe.db.get_value("Email Account", filters={"enable_automatic_linking": 1}):
 			return
 
 		for doctype, docname in parse_email([self.recipients, self.cc, self.bcc]):
@@ -615,9 +615,9 @@ def parse_email(email_strings):
 
 
 def get_email_without_link(email):
-	"""
-	returns email address without doctype links
-	returns admin@example.com for email admin+doctype+docname@example.com
+	"""Return email address without doctype links.
+
+	e.g. 'admin@example.com' is returned for email 'admin+doctype+docname@example.com'
 	"""
 	if not frappe.get_all("Email Account", filters={"enable_automatic_linking": 1}):
 		return email

frappe/core/doctype/communication/mixins.py

@@ -29,7 +29,7 @@ class CommunicationEmailMixin:
 		)
 
 	def get_email_with_displayname(self, email_address):
-		"""Returns email address after adding displayname."""
+		"""Return email address after adding displayname."""
 		display_name, email = parse_addr(email_address)
 		if display_name and display_name != email:
 			return email_address
@@ -151,7 +151,7 @@ class CommunicationEmailMixin:
 		return self.content
 
 	def get_attach_link(self, print_format):
-		"""Returns public link for the attachment via `templates/emails/print_link.html`."""
+		"""Return public link for the attachment via `templates/emails/print_link.html`."""
 		return frappe.get_template("templates/emails/print_link.html").render(
 			{
 				"url": get_url(),

frappe/core/doctype/data_import/data_import.js

@@ -136,47 +136,40 @@ frappe.ui.form.on("Data Import", {
 				let total_records = cint(r.message.total_records);
 
 				if (!total_records) return;
+				let action, message;
+				if (frm.doc.import_type === "Insert New Records") {
+					action = "imported";
+				} else {
+					action = "updated";
+				}
 
-				let message;
 				if (failed_records === 0) {
-					let message_args = [successful_records];
-					if (frm.doc.import_type === "Insert New Records") {
-						message =
-							successful_records > 1
-								? __("Successfully imported {0} records.", message_args)
-								: __("Successfully imported {0} record.", message_args);
+					let message_args = [action, successful_records];
+					if (successful_records === 1) {
+						message = __("Successfully {0} 1 record.", message_args);
 					} else {
-						message =
-							successful_records > 1
-								? __("Successfully updated {0} records.", message_args)
-								: __("Successfully updated {0} record.", message_args);
+						message = __("Successfully {0} {1} records.", message_args);
 					}
 				} else {
-					let message_args = [successful_records, total_records];
-					if (frm.doc.import_type === "Insert New Records") {
-						message =
-							successful_records > 1
-								? __(
-										"Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  )
-								: __(
-										"Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  );
+					let message_args = [action, successful_records, total_records];
+					if (successful_records === 1) {
+						message = __(
+							"Successfully {0} {1} record out of {2}. Click on Export Errored Rows, fix the errors and import again.",
+							message_args
+						);
 					} else {
-						message =
-							successful_records > 1
-								? __(
-										"Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  )
-								: __(
-										"Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  );
+						message = __(
+							"Successfully {0} {1} records out of {2}. Click on Export Errored Rows, fix the errors and import again.",
+							message_args
+						);
 					}
 				}
+
+				// If the job timed out, display an extra hint
+				if (r.message.status === "Timed Out") {
+					message += "<br/>" + __("Import timed out, please re-try.");
+				}
+
 				frm.dashboard.set_headline(message);
 			},
 		});

frappe/core/doctype/data_import/data_import.json

@@ -1,198 +1,198 @@
 {
-    "actions": [],
-    "autoname": "format:{reference_doctype} Import on {creation}",
-    "beta": 1,
-    "creation": "2019-08-04 14:16:08.318714",
-    "doctype": "DocType",
-    "editable_grid": 1,
-    "engine": "InnoDB",
-    "field_order": [
-        "reference_doctype",
-        "import_type",
-        "download_template",
-        "import_file",
-        "payload_count",
-        "html_5",
-        "google_sheets_url",
-        "refresh_google_sheet",
-        "column_break_5",
-        "status",
-        "submit_after_import",
-        "mute_emails",
-        "template_options",
-        "import_warnings_section",
-        "template_warnings",
-        "import_warnings",
-        "section_import_preview",
-        "import_preview",
-        "import_log_section",
-        "show_failed_logs",
-        "import_log_preview"
-    ],
-    "fields": [
-        {
-            "fieldname": "reference_doctype",
-            "fieldtype": "Link",
-            "in_list_view": 1,
-            "label": "Document Type",
-            "options": "DocType",
-            "reqd": 1,
-            "set_only_once": 1
-        },
-        {
-            "fieldname": "import_type",
-            "fieldtype": "Select",
-            "in_list_view": 1,
-            "label": "Import Type",
-            "options": "\nInsert New Records\nUpdate Existing Records",
-            "reqd": 1,
-            "set_only_once": 1
-        },
-        {
-            "depends_on": "eval:!doc.__islocal",
-            "fieldname": "import_file",
-            "fieldtype": "Attach",
-            "in_list_view": 1,
-            "label": "Import File",
-            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-        },
-        {
-            "fieldname": "import_preview",
-            "fieldtype": "HTML",
-            "label": "Import Preview"
-        },
-        {
-            "fieldname": "section_import_preview",
-            "fieldtype": "Section Break",
-            "label": "Preview"
-        },
-        {
-            "fieldname": "column_break_5",
-            "fieldtype": "Column Break"
-        },
-        {
-            "fieldname": "template_options",
-            "fieldtype": "Code",
-            "hidden": 1,
-            "label": "Template Options",
-            "options": "JSON",
-            "read_only": 1
-        },
-        {
-            "fieldname": "import_log_section",
-            "fieldtype": "Section Break",
-            "label": "Import Log"
-        },
-        {
-            "fieldname": "import_log_preview",
-            "fieldtype": "HTML",
-            "label": "Import Log Preview"
-        },
-        {
-            "default": "Pending",
-            "fieldname": "status",
-            "fieldtype": "Select",
-            "hidden": 1,
-            "label": "Status",
-            "no_copy": 1,
-            "options": "Pending\nSuccess\nPartial Success\nError",
-            "read_only": 1
-        },
-        {
-            "fieldname": "template_warnings",
-            "fieldtype": "Code",
-            "hidden": 1,
-            "label": "Template Warnings",
-            "options": "JSON"
-        },
-        {
-            "default": "0",
-            "fieldname": "submit_after_import",
-            "fieldtype": "Check",
-            "label": "Submit After Import",
-            "set_only_once": 1
-        },
-        {
-            "fieldname": "import_warnings_section",
-            "fieldtype": "Section Break",
-            "label": "Import File Errors and Warnings"
-        },
-        {
-            "fieldname": "import_warnings",
-            "fieldtype": "HTML",
-            "label": "Import Warnings"
-        },
-        {
-            "depends_on": "eval:!doc.__islocal",
-            "fieldname": "download_template",
-            "fieldtype": "Button",
-            "label": "Download Template"
-        },
-        {
-            "default": "1",
-            "fieldname": "mute_emails",
-            "fieldtype": "Check",
-            "label": "Don't Send Emails",
-            "set_only_once": 1
-        },
-        {
-            "default": "0",
-            "fieldname": "show_failed_logs",
-            "fieldtype": "Check",
-            "label": "Show Failed Logs"
-        },
-        {
-            "depends_on": "eval:!doc.__islocal && !doc.import_file",
-            "fieldname": "html_5",
-            "fieldtype": "HTML",
-            "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
-        },
-        {
-            "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
-            "description": "Must be a publicly accessible Google Sheets URL",
-            "fieldname": "google_sheets_url",
-            "fieldtype": "Data",
-            "label": "Import from Google Sheets",
-            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-        },
-        {
-            "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
-            "fieldname": "refresh_google_sheet",
-            "fieldtype": "Button",
-            "label": "Refresh Google Sheet"
-        },
-        {
-            "fieldname": "payload_count",
-            "fieldtype": "Int",
-            "hidden": 1,
-            "label": "Payload Count",
-            "read_only": 1
-        }
-    ],
-    "hide_toolbar": 1,
-    "links": [],
-    "modified": "2022-02-14 10:08:37.624914",
-    "modified_by": "Administrator",
-    "module": "Core",
-    "name": "Data Import",
-    "naming_rule": "Expression",
-    "owner": "Administrator",
-    "permissions": [
-        {
-            "create": 1,
-            "delete": 1,
-            "email": 1,
-            "export": 1,
-            "print": 1,
-            "read": 1,
-            "report": 1,
-            "role": "System Manager",
-            "share": 1,
-            "write": 1
-        }
-    ],
-    "sort_field": "modified",
-    "sort_order": "DESC",
-    "states": [],
-    "track_changes": 1
-}
+ "actions": [],
+ "autoname": "format:{reference_doctype} Import on {creation}",
+ "beta": 1,
+ "creation": "2019-08-04 14:16:08.318714",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "reference_doctype",
+  "import_type",
+  "download_template",
+  "import_file",
+  "payload_count",
+  "html_5",
+  "google_sheets_url",
+  "refresh_google_sheet",
+  "column_break_5",
+  "status",
+  "submit_after_import",
+  "mute_emails",
+  "template_options",
+  "import_warnings_section",
+  "template_warnings",
+  "import_warnings",
+  "section_import_preview",
+  "import_preview",
+  "import_log_section",
+  "show_failed_logs",
+  "import_log_preview"
+ ],
+ "fields": [
+  {
+   "fieldname": "reference_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Document Type",
+   "options": "DocType",
+   "reqd": 1,
+   "set_only_once": 1
+  },
+  {
+   "fieldname": "import_type",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Import Type",
+   "options": "\nInsert New Records\nUpdate Existing Records",
+   "reqd": 1,
+   "set_only_once": 1
+  },
+  {
+   "depends_on": "eval:!doc.__islocal",
+   "fieldname": "import_file",
+   "fieldtype": "Attach",
+   "in_list_view": 1,
+   "label": "Import File",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+  },
+  {
+   "fieldname": "import_preview",
+   "fieldtype": "HTML",
+   "label": "Import Preview"
+  },
+  {
+   "fieldname": "section_import_preview",
+   "fieldtype": "Section Break",
+   "label": "Preview"
+  },
+  {
+   "fieldname": "column_break_5",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "template_options",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "label": "Template Options",
+   "options": "JSON",
+   "read_only": 1
+  },
+  {
+   "fieldname": "import_log_section",
+   "fieldtype": "Section Break",
+   "label": "Import Log"
+  },
+  {
+   "fieldname": "import_log_preview",
+   "fieldtype": "HTML",
+   "label": "Import Log Preview"
+  },
+  {
+   "default": "Pending",
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "hidden": 1,
+   "label": "Status",
+   "no_copy": 1,
+   "options": "Pending\nSuccess\nPartial Success\nError\nTimed Out",
+   "read_only": 1
+  },
+  {
+   "fieldname": "template_warnings",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "label": "Template Warnings",
+   "options": "JSON"
+  },
+  {
+   "default": "0",
+   "fieldname": "submit_after_import",
+   "fieldtype": "Check",
+   "label": "Submit After Import",
+   "set_only_once": 1
+  },
+  {
+   "fieldname": "import_warnings_section",
+   "fieldtype": "Section Break",
+   "label": "Import File Errors and Warnings"
+  },
+  {
+   "fieldname": "import_warnings",
+   "fieldtype": "HTML",
+   "label": "Import Warnings"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal",
+   "fieldname": "download_template",
+   "fieldtype": "Button",
+   "label": "Download Template"
+  },
+  {
+   "default": "1",
+   "fieldname": "mute_emails",
+   "fieldtype": "Check",
+   "label": "Don't Send Emails",
+   "set_only_once": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "show_failed_logs",
+   "fieldtype": "Check",
+   "label": "Show Failed Logs"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && !doc.import_file",
+   "fieldname": "html_5",
+   "fieldtype": "HTML",
+   "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
+   "description": "Must be a publicly accessible Google Sheets URL",
+   "fieldname": "google_sheets_url",
+   "fieldtype": "Data",
+   "label": "Import from Google Sheets",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+  },
+  {
+   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
+   "fieldname": "refresh_google_sheet",
+   "fieldtype": "Button",
+   "label": "Refresh Google Sheet"
+  },
+  {
+   "fieldname": "payload_count",
+   "fieldtype": "Int",
+   "hidden": 1,
+   "label": "Payload Count",
+   "read_only": 1
+  }
+ ],
+ "hide_toolbar": 1,
+ "links": [],
+ "modified": "2023-12-15 12:45:49.452834",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Data Import",
+ "naming_rule": "Expression",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}

frappe/core/doctype/data_import/data_import.py

@@ -3,6 +3,8 @@
 
 import os
 
+from rq.timeouts import JobTimeoutException
+
 import frappe
 from frappe import _
 from frappe.core.doctype.data_import.exporter import Exporter
@@ -32,11 +34,13 @@ class DataImport(Document):
 		payload_count: DF.Int
 		reference_doctype: DF.Link
 		show_failed_logs: DF.Check
-		status: DF.Literal["Pending", "Success", "Partial Success", "Error"]
+		status: DF.Literal["Pending", "Success", "Partial Success", "Error", "Timed Out"]
 		submit_after_import: DF.Check
 		template_options: DF.Code | None
 		template_warnings: DF.Code | None
+
 	# end: auto-generated types
+
 	def validate(self):
 		doc_before_save = self.get_doc_before_save()
 		if (
@@ -136,6 +140,9 @@ def start_import(data_import):
 	try:
 		i = Importer(data_import.reference_doctype, data_import=data_import)
 		i.import_data()
+	except JobTimeoutException:
+		frappe.db.rollback()
+		data_import.db_set("status", "Timed Out")
 	except Exception:
 		frappe.db.rollback()
 		data_import.db_set("status", "Error")
@@ -190,6 +197,9 @@ def download_import_log(data_import_name):
 def get_import_status(data_import_name):
 	import_status = {}
 
+	data_import = frappe.get_doc("Data Import", data_import_name)
+	import_status["status"] = data_import.status
+
 	logs = frappe.get_all(
 		"Data Import Log",
 		fields=["count(*) as count", "success"],

frappe/core/doctype/data_import/data_import_list.js

@@ -20,13 +20,14 @@ frappe.listview_settings["Data Import"] = {
 			Success: "green",
 			"In Progress": "orange",
 			Error: "red",
+			"Timed Out": "orange",
 		};
 		let status = doc.status;
 
 		if (imports_in_progress.includes(doc.name)) {
 			status = "In Progress";
 		}
-		if (status == "Pending") {
+		if (status === "Pending") {
 			status = "Not Started";
 		}
 

frappe/core/doctype/data_import/importer.py

@@ -514,8 +514,8 @@ class ImportFile:
 
 	def parse_next_row_for_import(self, data):
 		"""
-		Parses rows that make up a doc. A doc maybe built from a single row or multiple rows.
-		Returns the doc, rows, and data without the rows.
+		Parse rows that make up a doc. A doc maybe built from a single row or multiple rows.
+		Return the doc, rows, and data without the rows.
 		"""
 		doctypes = self.header.doctypes
 

frappe/core/doctype/deleted_document/deleted_document.py

@@ -27,6 +27,14 @@ class DeletedDocument(Document):
 	# end: auto-generated types
 	pass
 
+	@staticmethod
+	def clear_old_logs(days=180):
+		from frappe.query_builder import Interval
+		from frappe.query_builder.functions import Now
+
+		table = frappe.qb.DocType("Deleted Document")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
+
 
 @frappe.whitelist()
 def restore(name, alert=True):

frappe/core/doctype/docfield/docfield.py

@@ -118,9 +118,10 @@ class DocField(Document):
 		width: DF.Data | None
 	# end: auto-generated types
 	def get_link_doctype(self):
-		"""Returns the Link doctype for the docfield (if applicable)
-		if fieldtype is Link: Returns "options"
-		if fieldtype is Table MultiSelect: Returns "options" of the Link field in the Child Table
+		"""Return the Link doctype for the `docfield` (if applicable).
+
+		* If fieldtype is Link: Return "options".
+		* If fieldtype is Table MultiSelect: Return "options" of the Link field in the Child Table.
 		"""
 		if self.fieldtype == "Link":
 			return self.options

frappe/core/doctype/doctype/doctype.js

@@ -3,7 +3,7 @@
 
 frappe.ui.form.on("DocType", {
 	onload: function (frm) {
-		if (frm.is_new()) {
+		if (frm.is_new() && !frm.doc?.fields) {
 			frappe.listview_settings["DocType"].new_doctype_dialog();
 		}
 	},

frappe/core/doctype/doctype/doctype.py

@@ -986,7 +986,7 @@ class DocType(Document):
 		add_column(self.name, "parentfield", "Data")
 
 	def get_max_idx(self):
-		"""Returns the highest `idx`"""
+		"""Return the highest `idx`."""
 		max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", self.name)
 		return max_idx and max_idx[0][0] or 0
 
@@ -1668,22 +1668,12 @@ def validate_permissions_for_doctype(doctype, for_remove=False, alert=False):
 
 
 def clear_permissions_cache(doctype):
+	from frappe.cache_manager import clear_user_cache
+
 	frappe.clear_cache(doctype=doctype)
 	delete_notification_count_for(doctype)
-	for user in frappe.db.sql_list(
-		"""
-		SELECT
-			DISTINCT `tabHas Role`.`parent`
-		FROM
-			`tabHas Role`,
-			`tabDocPerm`
-		WHERE `tabDocPerm`.`parent` = %s
-			AND `tabDocPerm`.`role` = `tabHas Role`.`role`
-			AND `tabHas Role`.`parenttype` = 'User'
-		""",
-		doctype,
-	):
-		frappe.clear_cache(user=user)
+
+	clear_user_cache()
 
 
 def validate_permissions(doctype, for_remove=False, alert=False):

frappe/core/doctype/doctype/test_doctype.py

@@ -786,6 +786,7 @@ def new_doctype(
 	depends_on: str = "",
 	fields: list[dict] | None = None,
 	custom: bool = True,
+	default: str | None = None,
 	**kwargs,
 ):
 	if not name:
@@ -803,6 +804,7 @@ def new_doctype(
 					"fieldname": "some_fieldname",
 					"fieldtype": "Data",
 					"unique": unique,
+					"default": default,
 					"depends_on": depends_on,
 				}
 			],

frappe/core/doctype/file/file.py

@@ -544,7 +544,7 @@ class File(Document):
 		return self._content
 
 	def get_full_path(self):
-		"""Returns file path from given file name"""
+		"""Return file path using the set file name."""
 
 		file_path = self.file_url or self.file_name
 
@@ -705,7 +705,7 @@ class File(Document):
 		return has_permission(self, "read")
 
 	def get_extension(self):
-		"""returns split filename and extension"""
+		"""Split and return filename and extension for the set `file_name`."""
 		return os.path.splitext(self.file_name)
 
 	def create_attachment_record(self):

frappe/core/doctype/page/page.py

@@ -118,7 +118,7 @@ class Page(Document):
 			shutil.rmtree(dir_path, ignore_errors=True)
 
 	def is_permitted(self):
-		"""Returns true if Has Role is not set or the user is allowed."""
+		"""Return True if `Has Role` is not set or the user is allowed."""
 		from frappe.utils import has_common
 
 		allowed = [

frappe/core/doctype/report/report.py

@@ -105,7 +105,7 @@ class Report(Document):
 				self.set("roles", roles)
 
 	def is_permitted(self):
-		"""Returns true if Has Role is not set or the user is allowed."""
+		"""Return True if `Has Role` is not set or the user is allowed."""
 		from frappe.utils import has_common
 
 		allowed = [

frappe/core/doctype/server_script/server_script.py

@@ -128,14 +128,14 @@ class ServerScript(Document):
 			frappe.msgprint(str(e), title=_("Compilation warning"))
 
 	def execute_method(self) -> dict:
-		"""Specific to API endpoint Server Scripts
+		"""Specific to API endpoint Server Scripts.
 
-		Raises:
-		        frappe.DoesNotExistError: If self.script_type is not API
-		        frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user
+		Raise:
+		        frappe.DoesNotExistError: If self.script_type is not API.
+		        frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user.
 
-		Returns:
-		        dict: Evaluates self.script with frappe.utils.safe_exec.safe_exec and returns the flags set in it's safe globals
+		Return:
+		        dict: Evaluate self.script with frappe.utils.safe_exec.safe_exec and return the flags set in its safe globals.
 		"""
 
 		if self.enable_rate_limit:
@@ -174,13 +174,13 @@ class ServerScript(Document):
 		safe_exec(self.script, script_filename=self.name)
 
 	def get_permission_query_conditions(self, user: str) -> list[str]:
-		"""Specific to Permission Query Server Scripts
+		"""Specific to Permission Query Server Scripts.
 
 		Args:
-		        user (str): Takes user email to execute script and return list of conditions
+		        user (str): Take user email to execute script and return list of conditions.
 
-		Returns:
-		        list: Returns list of conditions defined by rules in self.script
+		Return:
+		        list: Return list of conditions defined by rules in self.script.
 		"""
 		locals = {"user": user, "conditions": ""}
 		safe_exec(self.script, None, locals, script_filename=self.name)
@@ -189,12 +189,10 @@ class ServerScript(Document):
 
 	@frappe.whitelist()
 	def get_autocompletion_items(self):
-		"""Generates a list of a autocompletion strings from the context dict
+		"""Generate a list of autocompletion strings from the context dict
 		that is used while executing a Server Script.
 
-		Returns:
-		        list: Returns list of autocompletion items.
-		        For e.g., ["frappe.utils.cint", "frappe.get_all", ...]
+		e.g., ["frappe.utils.cint", "frappe.get_all", ...]
 		"""
 
 		def get_keys(obj):

frappe/core/doctype/sms_settings/sms_settings.py

@@ -46,7 +46,7 @@ def validate_receiver_nos(receiver_list):
 
 @frappe.whitelist()
 def get_contact_number(contact_name, ref_doctype, ref_name):
-	"returns mobile number of the contact"
+	"Return mobile number of the given contact."
 	number = frappe.db.sql(
 		"""select mobile_no, phone from tabContact
 		where name=%s

frappe/core/doctype/user/user.py

@@ -230,7 +230,7 @@ class User(Document):
 			frappe.cache.delete_key("users_for_mentions")
 
 	def has_website_permission(self, ptype, user, verbose=False):
-		"""Returns true if current user is the session user"""
+		"""Return True if current user is the session user."""
 		return self.name == frappe.session.user
 
 	def set_full_name(self):
@@ -686,7 +686,7 @@ class User(Document):
 		)
 
 	def get_blocked_modules(self):
-		"""Returns list of modules blocked for that user"""
+		"""Return list of modules blocked for that user."""
 		return [d.module for d in self.block_modules] if self.block_modules else []
 
 	def validate_user_email_inbox(self):
@@ -1083,7 +1083,7 @@ def user_query(doctype, txt, searchfield, start, page_len, filters):
 
 
 def get_total_users():
-	"""Returns total no. of system users"""
+	"""Return total number of system users."""
 	return flt(
 		frappe.db.sql(
 			"""SELECT SUM(`simultaneous_sessions`)
@@ -1118,7 +1118,7 @@ def get_system_users(exclude_users: Iterable[str] | str | None = None, limit: in
 
 
 def get_active_users():
-	"""Returns No. of system users who logged in, in the last 3 days"""
+	"""Return number of system users who logged in, in the last 3 days."""
 	return frappe.db.sql(
 		"""select count(*) from `tabUser`
 		where enabled = 1 and user_type != 'Website User'
@@ -1131,12 +1131,12 @@ def get_active_users():
 
 
 def get_website_users():
-	"""Returns total no. of website users"""
+	"""Return total number of website users."""
 	return frappe.db.count("User", filters={"enabled": True, "user_type": "Website User"})
 
 
 def get_active_website_users():
-	"""Returns No. of website users who logged in, in the last 3 days"""
+	"""Return number of website users who logged in, in the last 3 days."""
 	return frappe.db.sql(
 		"""select count(*) from `tabUser`
 		where enabled = 1 and user_type = 'Website User'

frappe/core/doctype/user_permission/user_permission.py

@@ -173,7 +173,7 @@ def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len,
 
 
 def get_permitted_documents(doctype):
-	"""Returns permitted documents from the given doctype for the session user"""
+	"""Return permitted documents from the given doctype for the session user."""
 	# sort permissions in a way to make the first permission in the list to be default
 	user_perm_list = sorted(
 		get_user_permissions().get(doctype, []), key=lambda x: x.get("is_default"), reverse=True

frappe/core/notifications.py

@@ -17,7 +17,7 @@ def get_notification_config():
 
 
 def get_things_todo(as_list=False):
-	"""Returns a count of incomplete todos"""
+	"""Return a count of incomplete ToDos."""
 	data = frappe.get_list(
 		"ToDo",
 		fields=["name", "description"] if as_list else "count(*)",
@@ -35,7 +35,7 @@ def get_things_todo(as_list=False):
 
 
 def get_todays_events(as_list: bool = False):
-	"""Returns a count of todays events in calendar"""
+	"""Return a count of today's events in calendar."""
 	from frappe.desk.doctype.event.event import get_events
 	from frappe.utils import nowdate
 

frappe/core/page/permission_manager/permission_manager.py

@@ -109,8 +109,10 @@ def add(parent, role, permlevel):
 
 
 @frappe.whitelist()
-def update(doctype, role, permlevel, ptype, value=None, if_owner=0):
-	"""Update role permission params
+def update(
+	doctype: str, role: str, permlevel: int, ptype: str, value=None, if_owner=0
+) -> str | None:
+	"""Update role permission params.
 
 	Args:
 	        doctype (str): Name of the DocType to update params for
@@ -119,8 +121,8 @@ def update(doctype, role, permlevel, ptype, value=None, if_owner=0):
 	        ptype (str): permission type, example "read", "delete", etc.
 	        value (None, optional): value for ptype, None indicates False
 
-	Returns:
-	        str: Refresh flag is permission is updated successfully
+	Return:
+	        str: Refresh flag if permission is updated successfully
 	"""
 
 	def clear_cache():

frappe/core/utils.py

@@ -7,7 +7,7 @@ import frappe
 
 
 def get_parent_doc(doc):
-	"""Returns document of `reference_doctype`, `reference_doctype`"""
+	"""Return document of `reference_doctype`, `reference_doctype`."""
 	if not hasattr(doc, "parent_doc"):
 		if doc.reference_doctype and doc.reference_name:
 			doc.parent_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name)
@@ -38,8 +38,7 @@ def set_timeline_doc(doc):
 
 
 def find(list_of_dict, match_function):
-	"""Returns a dict in a list of dicts on matching the conditions
-	        provided in match function
+	"""Return a dict in a list of dicts on matching the conditions provided in match function.
 
 	Usage:
 	        list_of_dict = [{'name': 'Suraj'}, {'name': 'Aditya'}]
@@ -54,8 +53,7 @@ def find(list_of_dict, match_function):
 
 
 def find_all(list_of_dict, match_function):
-	"""Returns all matching dicts in a list of dicts.
-	        Uses matching function to filter out the dicts
+	"""Return all matching dicts in a list of dicts. Uses matching function to filter out the dicts.
 
 	Usage:
 	        colored_shapes = [
@@ -86,6 +84,7 @@ def ljust_list(_list, length, fill_word=None):
 	return _list
 
 
-def html2text(html, strip_links=False, wrap=True):
+def html2text(html: str, strip_links=False, wrap=True) -> str:
+	"""Return the given `html` as markdown text."""
 	strip = ["a"] if strip_links else None
 	return md(html, heading_style="ATX", strip=strip, wrap=wrap)

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -240,8 +240,9 @@ class TestCustomizeForm(FrappeTestCase):
 		# Using Notification Log doctype as it doesn't have any other custom fields
 		d = self.get_customize_form("Notification Log")
 
+		new_document_length = 255
 		document_name = d.get("fields", {"fieldname": "document_name"})[0]
-		document_name.length = 255
+		document_name.length = new_document_length
 		d.run_method("save_customization")
 
 		self.assertEqual(
@@ -250,11 +251,9 @@ class TestCustomizeForm(FrappeTestCase):
 				{"doc_type": "Notification Log", "property": "length", "field_name": "document_name"},
 				"value",
 			),
-			"255",
+			str(new_document_length),
 		)
 
-		self.assertTrue(d.flags.update_db)
-
 		length = frappe.db.sql(
 			"""SELECT character_maximum_length
 			FROM information_schema.columns
@@ -262,7 +261,7 @@ class TestCustomizeForm(FrappeTestCase):
 			AND column_name = 'document_name'"""
 		)[0][0]
 
-		self.assertEqual(length, 255)
+		self.assertEqual(length, new_document_length)
 
 	def test_custom_link(self):
 		try:

frappe/database/database.py

@@ -123,7 +123,7 @@ class Database:
 		self._conn.select_db(db_name)
 
 	def get_connection(self):
-		"""Returns a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects"""
+		"""Return a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects."""
 		raise NotImplementedError
 
 	def get_database_size(self):
@@ -160,7 +160,7 @@ class Database:
 		:param ignore_ddl: Catch exception if table, column missing.
 		:param auto_commit: Commit after executing the query.
 		:param update: Update this dict to all rows (if returned `as_dict`).
-		:param run: Returns query without executing it if False.
+		:param run: Return query without executing it if False.
 		:param pluck: Get the plucked field only.
 		:param explain: Print `EXPLAIN` in error log.
 		Examples:
@@ -397,7 +397,7 @@ class Database:
 			raise ImplicitCommitError("This statement can cause implicit commit", query)
 
 	def fetch_as_dict(self) -> list[frappe._dict]:
-		"""Internal. Converts results to dict."""
+		"""Internal. Convert results to dict."""
 		result = self.last_result
 		if result:
 			keys = [column[0] for column in self._cursor.description]
@@ -410,7 +410,7 @@ class Database:
 			frappe.cache.delete_key("db_tables")
 
 	def get_description(self):
-		"""Returns result metadata."""
+		"""Return result metadata."""
 		return self._cursor.description
 
 	@staticmethod
@@ -419,7 +419,7 @@ class Database:
 		return [[value for value in row] for row in res]
 
 	def get(self, doctype, filters=None, as_dict=True, cache=False):
-		"""Returns `get_value` with fieldname='*'"""
+		"""Return `get_value` with fieldname='*'."""
 		return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache)
 
 	def get_value(
@@ -438,7 +438,7 @@ class Database:
 		pluck=False,
 		distinct=False,
 	):
-		"""Returns a document property or list of properties.
+		"""Return a document property or list of properties.
 
 		:param doctype: DocType name.
 		:param filters: Filters like `{"x":"y"}` or name of the document. `None` if Single DocType.
@@ -510,7 +510,7 @@ class Database:
 		distinct=False,
 		limit=None,
 	):
-		"""Returns multiple document properties.
+		"""Return multiple document properties.
 
 		:param doctype: DocType name.
 		:param filters: Filters like `{"x":"y"}` or name of the document.
@@ -926,11 +926,11 @@ class Database:
 		self.set_default(key, val, user)
 
 	def get_global(self, key, user="__global"):
-		"""Returns a global key value."""
+		"""Return a global key value."""
 		return self.get_default(key, user)
 
 	def get_default(self, key, parent="__default"):
-		"""Returns default value as a list if multiple or single"""
+		"""Return default value as a list if multiple or single."""
 		d = self.get_defaults(key, parent)
 		return isinstance(d, list) and d[0] or d
 
@@ -1006,7 +1006,7 @@ class Database:
 		return self.exists("DocField", {"fieldname": fn, "parent": dt})
 
 	def table_exists(self, doctype, cached=True):
-		"""Returns True if table for given doctype exists."""
+		"""Return True if table for given doctype exists."""
 		return f"tab{doctype}" in self.get_tables(cached=cached)
 
 	def has_table(self, doctype):
@@ -1016,7 +1016,7 @@ class Database:
 		raise NotImplementedError
 
 	def a_row_exists(self, doctype):
-		"""Returns True if atleast one row exists."""
+		"""Return True if at least one row exists."""
 		return frappe.get_all(doctype, limit=1, order_by=None, as_list=True)
 
 	def exists(self, dt, dn=None, cache=False):
@@ -1055,7 +1055,7 @@ class Database:
 		return self.get_value(dt, dn, ignore=True, cache=cache, order_by=None)
 
 	def count(self, dt, filters=None, debug=False, cache=False, distinct: bool = True):
-		"""Returns `COUNT(*)` for given DocType and filters."""
+		"""Return `COUNT(*)` for given DocType and filters."""
 		if cache and not filters:
 			cache_count = frappe.cache.get_value(f"doctype:count:{dt}")
 			if cache_count is not None:
@@ -1098,7 +1098,7 @@ class Database:
 		)
 
 	def get_db_table_columns(self, table) -> list[str]:
-		"""Returns list of column names from given table."""
+		"""Return list of column names from given table."""
 		columns = frappe.cache.hget("table_columns", table)
 		if columns is None:
 			information_schema = frappe.qb.Schema("information_schema")
@@ -1116,14 +1116,14 @@ class Database:
 		return columns
 
 	def get_table_columns(self, doctype):
-		"""Returns list of column names from given doctype."""
+		"""Return list of column names from given doctype."""
 		columns = self.get_db_table_columns("tab" + doctype)
 		if not columns:
 			raise self.TableMissingError("DocType", doctype)
 		return columns
 
 	def has_column(self, doctype, column):
-		"""Returns True if column exists in database."""
+		"""Return True if column exists in database."""
 		return column in self.get_table_columns(doctype)
 
 	def has_index(self, table_name, index_name):

frappe/database/db_manager.py

@@ -57,14 +57,15 @@ class DbManager:
 		from frappe.database import get_command
 		from frappe.utils import execute_in_shell
 
-		pv = which("pv")
-
 		command = []
 
-		if pv:
-			command.extend([pv, source, "|"])
-			source = []
-			print("Restoring Database file...")
+		if source.endswith(".gz"):
+			if gzip := which("gzip"):
+				command.extend([gzip, "-cd", source, "|"])
+				source = []
+			else:
+				raise Exception("`gzip` not installed")
+
 		else:
 			source = ["<", source]
 

frappe/database/mariadb/database.py

@@ -191,7 +191,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		}
 
 	def get_database_size(self):
-		"""'Returns database size in MB"""
+		"""Return database size in MB."""
 		db_size = self.sql(
 			"""
 			SELECT `table_schema` as `database_name`,
@@ -314,7 +314,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		return "ON DUPLICATE key UPDATE "
 
 	def get_table_columns_description(self, table_name):
-		"""Returns list of column and its description"""
+		"""Return list of columns with descriptions."""
 		return self.sql(
 			"""select
 			column_name as 'name',
@@ -339,7 +339,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		)
 
 	def get_column_type(self, doctype, column):
-		"""Returns column type from database."""
+		"""Return column type from database."""
 		information_schema = frappe.qb.Schema("information_schema")
 		table = get_table_name(doctype)
 
@@ -440,13 +440,13 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 
 			self.commit()
 			db_table.sync()
-			self.begin()
+			self.commit()
 
 	def get_database_list(self):
 		return self.sql("SHOW DATABASES", pluck=True)
 
 	def get_tables(self, cached=True):
-		"""Returns list of tables"""
+		"""Return list of tables."""
 		to_query = not cached
 
 		if cached:

frappe/database/operator_map.py

@@ -17,21 +17,21 @@ def like(key: Field, value: str) -> frappe.qb:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `LIKE`
+	Return:
+	        frappe.qb: `frappe.qb` object with `LIKE`
 	"""
 	return key.like(value)
 
 
 def func_in(key: Field, value: list | tuple) -> frappe.qb:
-	"""Wrapper method for `IN`
+	"""Wrapper method for `IN`.
 
 	Args:
 	        key (str): field
 	        value (Union[int, str]): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `IN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `IN`
 	"""
 	if isinstance(value, str):
 		value = value.split(",")
@@ -39,27 +39,27 @@ def func_in(key: Field, value: list | tuple) -> frappe.qb:
 
 
 def not_like(key: Field, value: str) -> frappe.qb:
-	"""Wrapper method for `NOT LIKE`
+	"""Wrapper method for `NOT LIKE`.
 
 	Args:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `NOT LIKE`
+	Return:
+	        frappe.qb: `frappe.qb` object with `NOT LIKE`
 	"""
 	return key.not_like(value)
 
 
 def func_not_in(key: Field, value: list | tuple | str):
-	"""Wrapper method for `NOT IN`
+	"""Wrapper method for `NOT IN`.
 
 	Args:
 	        key (str): field
 	        value (Union[int, str]): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `NOT IN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `NOT IN`
 	"""
 	if isinstance(value, str):
 		value = value.split(",")
@@ -73,21 +73,21 @@ def func_regex(key: Field, value: str) -> frappe.qb:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `REGEX`
+	Return:
+	        frappe.qb: `frappe.qb` object with `REGEX`
 	"""
 	return key.regex(value)
 
 
 def func_between(key: Field, value: list | tuple) -> frappe.qb:
-	"""Wrapper method for `BETWEEN`
+	"""Wrapper method for `BETWEEN`.
 
 	Args:
 	        key (str): field
 	        value (Union[int, str]): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `BETWEEN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `BETWEEN`
 	"""
 	return key[slice(*value)]
 
@@ -98,14 +98,14 @@ def func_is(key, value):
 
 
 def func_timespan(key: Field, value: str) -> frappe.qb:
-	"""Wrapper method for `TIMESPAN`
+	"""Wrapper method for `TIMESPAN`.
 
 	Args:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `TIMESPAN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `TIMESPAN`
 	"""
 
 	return func_between(key, get_timespan_date_range(value))

frappe/database/postgres/database.py

@@ -197,7 +197,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 		return str(psycopg2.extensions.QuotedString(s))
 
 	def get_database_size(self):
-		"""'Returns database size in MB"""
+		"""Return database size in MB"""
 		db_size = self.sql(
 			"SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", self.db_name, as_dict=True
 		)
@@ -329,7 +329,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 
 			self.commit()
 			db_table.sync()
-			self.begin()
+			self.commit()
 
 	@staticmethod
 	def get_on_duplicate_update(key="name"):
@@ -380,7 +380,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 			)
 
 	def get_table_columns_description(self, table_name):
-		"""Returns list of column and its description"""
+		"""Return list of columns with description."""
 		# pylint: disable=W1401
 		return self.sql(
 			"""
@@ -411,7 +411,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 		)
 
 	def get_column_type(self, doctype, column):
-		"""Returns column type from database."""
+		"""Return column type from database."""
 		information_schema = frappe.qb.Schema("information_schema")
 		table = get_table_name(doctype)
 

frappe/database/postgres/setup_db.py

@@ -1,7 +1,7 @@
 import os
 
 import frappe
-from frappe import _
+from frappe.database.db_manager import DbManager
 
 
 def setup_database():
@@ -36,45 +36,16 @@ def bootstrap_database(db_name, verbose, source_sql=None):
 
 
 def import_db_from_sql(source_sql=None, verbose=False):
-	import shlex
-	from shutil import which
-
-	from frappe.database import get_command
-	from frappe.utils import execute_in_shell
-
-	# bootstrap db
+	if verbose:
+		print("Starting database import...")
+	db_name = frappe.conf.db_name
 	if not source_sql:
 		source_sql = os.path.join(os.path.dirname(__file__), "framework_postgres.sql")
-
-	pv = which("pv")
-
-	command = []
-
-	if pv:
-		command.extend([pv, source_sql, "|"])
-		source = []
-		print("Restoring Database file...")
-	else:
-		source = ["-f", source_sql]
-
-	bin, args, bin_name = get_command(
-		host=frappe.conf.db_host,
-		port=frappe.conf.db_port,
-		user=frappe.conf.db_name,
-		password=frappe.conf.db_password,
-		db_name=frappe.conf.db_name,
+	DbManager(frappe.local.db).restore_database(
+		verbose, db_name, source_sql, db_name, frappe.conf.db_password
 	)
-
-	if not bin:
-		frappe.throw(
-			_("{} not found in PATH! This is required to restore the database.").format(bin_name),
-			exc=frappe.ExecutableNotFound,
-		)
-	command.append(bin)
-	command.append(shlex.join(args))
-	command.extend(source)
-	execute_in_shell(" ".join(command), check_exit_code=True, verbose=verbose)
-	frappe.cache.delete_keys("")  # Delete all keys associated with this site.
+	if verbose:
+		print("Imported from database %s" % source_sql)
 
 
 def get_root_connection(root_login=None, root_password=None):

frappe/database/query.py

@@ -218,7 +218,7 @@ class Engine:
 			self.query = self.query.where(operator_fn(_field, _value))
 
 	def get_function_object(self, field: str) -> "Function":
-		"""Expects field to look like 'SUM(*)' or 'name' or something similar. Returns PyPika Function object"""
+		"""Return PyPika Function object. Expect field to look like 'SUM(*)' or 'name' or something similar."""
 		func = field.split("(", maxsplit=1)[0].capitalize()
 		args_start, args_end = len(func) + 1, field.index(")")
 		args = field[args_start:args_end].split(",")

frappe/defaults.py

@@ -79,7 +79,7 @@ def is_a_user_permission_key(key):
 
 
 def not_in_user_permission(key, value, user=None):
-	# returns true or false based on if value exist in user permission
+	# return true or false based on if value exist in user permission
 	user = user or frappe.session.user
 	user_permission = get_user_permissions(user).get(frappe.unscrub(key)) or []
 

frappe/desk/calendar.py

@@ -19,7 +19,7 @@ def update_event(args, field_map):
 
 
 def get_event_conditions(doctype, filters=None):
-	"""Returns SQL conditions with user permissions and filters for event queries"""
+	"""Return SQL conditions with user permissions and filters for event queries."""
 	from frappe.desk.reportview import get_filters_cond
 
 	if not frappe.has_permission(doctype):

frappe/desk/desktop.py

@@ -68,7 +68,7 @@ class Workspace:
 		)
 
 	def is_permitted(self):
-		"""Returns true if Has Role is not set or the user is allowed."""
+		"""Return true if `Has Role` is not set or the user is allowed."""
 		from frappe.utils import has_common
 
 		allowed = [d.role for d in self.doc.roles]
@@ -383,13 +383,12 @@ class Workspace:
 @frappe.whitelist()
 @frappe.read_only()
 def get_desktop_page(page):
-	"""Applies permissions, customizations and returns the configruration for a page
-	on desk.
+	"""Apply permissions, customizations and return the configuration for a page on desk.
 
 	Args:
 	        page (json): page data
 
-	Returns:
+	Return:
 	        dict: dictionary of cards, charts and shortcuts to be displayed on website
 	"""
 	try:
@@ -503,7 +502,7 @@ def get_custom_doctype_list(module):
 
 
 def get_custom_report_list(module):
-	"""Returns list on new style reports for modules."""
+	"""Return list on new style reports for modules."""
 	reports = frappe.get_all(
 		"Report",
 		fields=["name", "ref_doctype", "report_type"],
@@ -617,14 +616,14 @@ def new_widget(config, doctype, parentfield):
 
 
 def prepare_widget(config, doctype, parentfield):
-	"""Create widget child table entries with parent details
+	"""Create widget child table entries with parent details.
 
 	Args:
 	        config (dict): Dictionary containing widget config
 	        doctype (string): Doctype name of the child table
 	        parentfield (string): Parent field for the child table
 
-	Returns:
+	Return:
 	        TYPE: List of Document objects
 	"""
 	if not config:

frappe/desk/doctype/tag/tag.py

@@ -77,15 +77,15 @@ class DocTags:
 		self.dt = dt
 
 	def get_tag_fields(self):
-		"""returns tag_fields property"""
+		"""Return `tag_fields` property."""
 		return frappe.db.get_value("DocType", self.dt, "tag_fields")
 
 	def get_tags(self, dn):
-		"""returns tag for a particular item"""
+		"""Return tag for a particular item."""
 		return (frappe.db.get_value(self.dt, dn, "_user_tags", ignore=1) or "").strip()
 
 	def add(self, dn, tag):
-		"""add a new user tag"""
+		"""Add a new user tag."""
 		tl = self.get_tags(dn).split(",")
 		if tag not in tl:
 			tl.append(tag)
@@ -94,16 +94,16 @@ class DocTags:
 			self.update(dn, tl)
 
 	def remove(self, dn, tag):
-		"""remove a user tag"""
+		"""Remove a user tag."""
 		tl = self.get_tags(dn).split(",")
 		self.update(dn, filter(lambda x: x.lower() != tag.lower(), tl))
 
 	def remove_all(self, dn):
-		"""remove all user tags (call before delete)"""
+		"""Remove all user tags (call before delete)."""
 		self.update(dn, [])
 
 	def update(self, dn, tl):
-		"""updates the _user_tag column in the table"""
+		"""Update the `_user_tag` column in the table."""
 
 		if not tl:
 			tags = ""
@@ -128,16 +128,15 @@ class DocTags:
 				raise
 
 	def setup(self):
-		"""adds the _user_tags column if not exists"""
+		"""Add the `_user_tags` column if not exists."""
 		from frappe.database.schema import add_column
 
 		add_column(self.dt, "_user_tags", "Data")
 
 
 def delete_tags_for_document(doc):
-	"""
-	Delete the Tag Link entry of a document that has
-	been deleted
+	"""Delete the Tag Link entry of a document that has been deleted.
+
 	:param doc: Deleted document
 	"""
 	if not frappe.db.table_exists("Tag Link"):
@@ -147,7 +146,7 @@ def delete_tags_for_document(doc):
 
 
 def update_tags(doc, tags):
-	"""Adds tags for documents
+	"""Add tags for documents.
 
 	:param doc: Document to be added to global tags
 	"""
@@ -181,8 +180,8 @@ def update_tags(doc, tags):
 
 @frappe.whitelist()
 def get_documents_for_tag(tag):
-	"""
-	Search for given text in Tag Link
+	"""Search for given text in Tag Link.
+
 	:param tag: tag to be searched
 	"""
 	# remove hastag `#` from tag

frappe/desk/doctype/todo/todo.py

@@ -136,7 +136,7 @@ class ToDo(Document):
 
 	@classmethod
 	def get_owners(cls, filters=None):
-		"""Returns list of owners after applying filters on todo's."""
+		"""Return list of owners after applying filters on ToDos."""
 		rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=["allocated_to"])
 		return [parse_addr(row.allocated_to)[1] for row in rows if row.allocated_to]
 

frappe/desk/form/linked_with.py

@@ -131,7 +131,7 @@ class SubmittableDocumentTree:
 		return self._references_across_doctypes.get(doctype, [])
 
 	def get_document_sources(self):
-		"""Returns list of doctypes from where we access submittable documents."""
+		"""Return list of doctypes from where we access submittable documents."""
 		return list(set(self.get_link_sources() + [self.root_doctype]))
 
 	def get_link_sources(self):
@@ -139,7 +139,7 @@ class SubmittableDocumentTree:
 		return list(set(self.get_submittable_doctypes()) - set(get_exempted_doctypes() or []))
 
 	def get_submittable_doctypes(self) -> list[str]:
-		"""Returns list of submittable doctypes."""
+		"""Return list of submittable doctypes."""
 		if not self._submittable_doctypes:
 			self._submittable_doctypes = frappe.get_all(
 				"DocType", {"is_submittable": 1}, pluck="name", order_by=None
@@ -148,7 +148,7 @@ class SubmittableDocumentTree:
 
 
 def get_child_tables_of_doctypes(doctypes: list[str] = None):
-	"""Returns child tables by doctype."""
+	"""Return child tables by doctype."""
 	filters = [["fieldtype", "=", "Table"]]
 	filters_for_docfield = filters
 	filters_for_customfield = filters
@@ -387,7 +387,7 @@ def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=None):
 	        docinfo (dict): The document to check for submitted and non-exempt from auto-cancel
 	        ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling.
 
-	Returns:
+	Return:
 	        bool: True if linked document passes all validations, else False
 	"""
 	# ignore doctype to cancel

frappe/desk/form/load.py

@@ -274,7 +274,7 @@ def _get_communications(doctype, name, start=0, limit=20):
 def get_communication_data(
 	doctype, name, start=0, limit=20, after=None, fields=None, group_by=None, as_dict=True
 ):
-	"""Returns list of communications for a given document"""
+	"""Return list of communications for a given document."""
 	if not fields:
 		fields = """
 			C.name, C.communication_type, C.communication_medium,

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -349,12 +349,16 @@ frappe.setup.SetupWizardSlide = class SetupWizardSlide extends frappe.ui.Slide {
 	setup_telemetry_events() {
 		let me = this;
 		this.fields.filter(frappe.model.is_value_type).forEach((field) => {
-			me.get_input(field.fieldname).on("change", function () {
-				frappe.telemetry.capture(`${field.fieldname}_set`, "setup");
-				if (field.fieldname == "enable_telemetry" && !me.get_value("enable_telemetry")) {
-					frappe.telemetry.disable();
-				}
-			});
+			field.fieldname &&
+				me.get_input(field.fieldname)?.on("change", function () {
+					frappe.telemetry.capture(`${field.fieldname}_set`, "setup");
+					if (
+						field.fieldname == "enable_telemetry" &&
+						!me.get_value("enable_telemetry")
+					) {
+						frappe.telemetry.disable();
+					}
+				});
 		});
 	}
 };

frappe/desk/query_report.py

@@ -123,7 +123,7 @@ def generate_report_result(
 
 
 def normalize_result(result, columns):
-	# Converts to list of dicts from list of lists/tuples
+	# Convert to list of dicts from list of lists/tuples
 	data = []
 	column_names = [column["fieldname"] for column in columns]
 	if result and isinstance(result[0], (list, tuple)):
@@ -318,6 +318,7 @@ def export_query():
 	file_format_type = form_params.file_format_type
 	custom_columns = frappe.parse_json(form_params.custom_columns or "[]")
 	include_indentation = form_params.include_indentation
+	include_filters = form_params.include_filters
 	visible_idx = form_params.visible_idx
 
 	if isinstance(visible_idx, str):
@@ -327,6 +328,8 @@ def export_query():
 		report_name, form_params.filters, custom_columns=custom_columns, are_default_filters=False
 	)
 	data = frappe._dict(data)
+	data.filters = form_params.applied_filters
+
 	if not data.columns:
 		frappe.respond_as_web_page(
 			_("No data to export"),
@@ -335,7 +338,9 @@ def export_query():
 		return
 
 	format_duration_fields(data)
-	xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
+	xlsx_data, column_widths = build_xlsx_data(
+		data, visible_idx, include_indentation, include_filters=include_filters
+	)
 
 	if file_format_type == "CSV":
 		content = get_csv_bytes(xlsx_data, csv_params)
@@ -360,7 +365,9 @@ def format_duration_fields(data: frappe._dict) -> None:
 				row[index] = format_duration(row[index])
 
 
-def build_xlsx_data(data, visible_idx, include_indentation, ignore_visible_idx=False):
+def build_xlsx_data(
+	data, visible_idx, include_indentation, include_filters=False, ignore_visible_idx=False
+):
 	EXCEL_TYPES = (
 		str,
 		bool,
@@ -380,17 +387,34 @@ def build_xlsx_data(data, visible_idx, include_indentation, ignore_visible_idx=F
 		# Note: converted for faster lookups
 		visible_idx = set(visible_idx)
 
-	result = [[]]
+	result = []
 	column_widths = []
 
+	if cint(include_filters):
+		filter_data = []
+		filters = data.filters
+		for filter_name, filter_value in filters.items():
+			if not filter_value:
+				continue
+			filter_value = (
+				", ".join([cstr(x) for x in filter_value])
+				if isinstance(filter_value, list)
+				else cstr(filter_value)
+			)
+			filter_data.append([cstr(filter_name), filter_value])
+		filter_data.append([])
+		result += filter_data
+
+	column_data = []
 	for column in data.columns:
 		if column.get("hidden"):
 			continue
-		result[0].append(_(column.get("label")))
+		column_data.append(_(column.get("label")))
 		column_width = cint(column.get("width", 0))
 		# to convert into scale accepted by openpyxl
 		column_width /= 10
 		column_widths.append(column_width)
+	result.append(column_data)
 
 	# build table from result
 	for row_idx, row in enumerate(data.result):
@@ -603,11 +627,11 @@ def has_match(
 	columns_dict,
 	user,
 ):
-	"""Returns True if after evaluating permissions for each linked doctype
-	- There is an owner match for the ref_doctype
-	- `and` There is a user permission match for all linked doctypes
+	"""Return True if after evaluating permissions for each linked doctype:
+	        - There is an owner match for the ref_doctype
+	        - `and` There is a user permission match for all linked doctypes
 
-	Returns True if the row is empty
+	Return True if the row is empty.
 
 	Note:
 	Each doctype could have multiple conflicting user permission doctypes.
@@ -705,9 +729,10 @@ def get_linked_doctypes(columns, data):
 
 
 def get_columns_dict(columns):
-	"""Returns a dict with column docfield values as dict
+	"""Return a dict with column docfield values as dict.
+
 	The keys for the dict are both idx and fieldname,
-	so either index or fieldname can be used to search for a column's docfield properties
+	so either index or fieldname can be used to search for a column's docfield properties.
 	"""
 	columns_dict = frappe._dict()
 	for idx, col in enumerate(columns):

frappe/desk/reportview.py

@@ -217,6 +217,8 @@ def clean_params(data):
 def parse_json(data):
 	if (filters := data.get("filters")) and isinstance(filters, str):
 		data["filters"] = json.loads(filters)
+	if (applied_filters := data.get("applied_filters")) and isinstance(applied_filters, str):
+		data["applied_filters"] = json.loads(applied_filters)
 	if (or_filters := data.get("or_filters")) and isinstance(or_filters, str):
 		data["or_filters"] = json.loads(or_filters)
 	if (fields := data.get("fields")) and isinstance(fields, str):

frappe/desk/treeview.py

@@ -15,8 +15,7 @@ def get_all_nodes(doctype, label, parent, tree_method, **filters):
 
 	tree_method = frappe.get_attr(tree_method)
 
-	if tree_method not in frappe.whitelisted:
-		frappe.throw(_("Not Permitted"), frappe.PermissionError)
+	frappe.is_whitelisted(tree_method)
 
 	data = tree_method(doctype, parent, **filters)
 	out = [dict(parent=label, data=data)]

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -128,7 +128,7 @@ class AutoEmailReport(Document):
 			)
 
 	def get_report_content(self):
-		"""Returns file in for the report in given format"""
+		"""Return file for the report in given format."""
 		report = frappe.get_doc("Report", self.report)
 
 		self.filters = frappe.parse_json(self.filters) if self.filters else {}

frappe/email/doctype/email_account/email_account.py

@@ -237,7 +237,7 @@ class EmailAccount(Document):
 		return frappe.db.get_value("Email Domain", domain, EMAIL_DOMAIN_FIELDS, as_dict=True)
 
 	def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"):
-		"""Returns logged in POP3/IMAP connection object."""
+		"""Return logged in POP3/IMAP connection object."""
 		oauth_token = self.get_oauth_token()
 		args = frappe._dict(
 			{

frappe/email/doctype/email_group/email_group.js

@@ -1,74 +1,97 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on("Email Group", "refresh", function (frm) {
-	if (!frm.is_new()) {
-		frm.add_custom_button(
-			__("Import Subscribers"),
-			function () {
-				frappe.prompt(
-					{
-						fieldtype: "Select",
-						options: frm.doc.__onload.import_types,
-						label: __("Import Email From"),
-						fieldname: "doctype",
-						reqd: 1,
-					},
-					function (data) {
-						frappe.call({
-							method: "frappe.email.doctype.email_group.email_group.import_from",
-							args: {
-								name: frm.doc.name,
-								doctype: data.doctype,
-							},
-							callback: function (r) {
-								frm.set_value("total_subscribers", r.message);
-							},
-						});
-					},
-					__("Import Subscribers"),
-					__("Import")
-				);
-			},
-			__("Action")
-		);
+frappe.ui.form.on("Email Group", {
+	refresh: function (frm) {
+		if (!frm.is_new()) {
+			frm.add_custom_button(
+				__("Import Subscribers"),
+				function () {
+					frappe.prompt(
+						{
+							fieldtype: "Select",
+							options: frm.doc.__onload.import_types,
+							label: __("Import Email From"),
+							fieldname: "doctype",
+							reqd: 1,
+						},
+						function (data) {
+							frappe.call({
+								method: "frappe.email.doctype.email_group.email_group.import_from",
+								args: {
+									name: frm.doc.name,
+									doctype: data.doctype,
+								},
+								callback: function (r) {
+									frm.set_value("total_subscribers", r.message);
+								},
+							});
+						},
+						__("Import Subscribers"),
+						__("Import")
+					);
+				},
+				__("Action")
+			);
 
-		frm.add_custom_button(
-			__("Add Subscribers"),
-			function () {
-				frappe.prompt(
-					{
-						fieldtype: "Text",
-						label: __("Email Addresses"),
-						fieldname: "email_list",
-						reqd: 1,
-					},
-					function (data) {
-						frappe.call({
-							method: "frappe.email.doctype.email_group.email_group.add_subscribers",
-							args: {
-								name: frm.doc.name,
-								email_list: data.email_list,
-							},
-							callback: function (r) {
-								frm.set_value("total_subscribers", r.message);
-							},
-						});
-					},
-					__("Add Subscribers"),
-					__("Add")
-				);
-			},
-			__("Action")
-		);
+			frm.add_custom_button(
+				__("Add Subscribers"),
+				function () {
+					frappe.prompt(
+						{
+							fieldtype: "Text",
+							label: __("Email Addresses"),
+							fieldname: "email_list",
+							reqd: 1,
+						},
+						function (data) {
+							frappe.call({
+								method: "frappe.email.doctype.email_group.email_group.add_subscribers",
+								args: {
+									name: frm.doc.name,
+									email_list: data.email_list,
+								},
+								callback: function (r) {
+									frm.set_value("total_subscribers", r.message);
+								},
+							});
+						},
+						__("Add Subscribers"),
+						__("Add")
+					);
+				},
+				__("Action")
+			);
 
-		frm.add_custom_button(
-			__("New Newsletter"),
-			function () {
-				frappe.route_options = { email_group: frm.doc.name };
-				frappe.new_doc("Newsletter");
-			},
-			__("Action")
-		);
-	}
+			frm.add_custom_button(
+				__("New Newsletter"),
+				function () {
+					frappe.route_options = { email_group: frm.doc.name };
+					frappe.new_doc("Newsletter");
+				},
+				__("Action")
+			);
+		}
+
+		frm.trigger("preview_welcome_url");
+	},
+	welcome_url(frm) {
+		frm.trigger("preview_welcome_url");
+	},
+	add_query_parameters: function (frm) {
+		frm.trigger("preview_welcome_url");
+	},
+	preview_welcome_url: function (frm) {
+		if (frm.doc.add_query_parameters && frm.doc.welcome_url) {
+			frm.call("preview_welcome_url", { email: "mail@example.org" }).then((r) => {
+				frm.set_df_property(
+					"add_query_parameters",
+					"description",
+					`${__("Preview:")} ${r.message}`
+				);
+			});
+		} else {
+			frm.set_df_property("add_query_parameters", "description", "");
+		}
+	},
 });

frappe/email/doctype/email_group/email_group.json

@@ -9,9 +9,13 @@
  "engine": "InnoDB",
  "field_order": [
   "title",
+  "column_break_oyyj",
   "total_subscribers",
+  "sign_up_and_confirmation_section",
   "confirmation_email_template",
-  "welcome_email_template"
+  "welcome_email_template",
+  "welcome_url",
+  "add_query_parameters"
  ],
  "fields": [
   {
@@ -41,6 +45,29 @@
    "fieldtype": "Link",
    "label": "Welcome Email Template",
    "options": "Email Template"
+  },
+  {
+   "fieldname": "column_break_oyyj",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "sign_up_and_confirmation_section",
+   "fieldtype": "Section Break",
+   "label": "Sign Up and Confirmation"
+  },
+  {
+   "description": "Redirect to this URL after successful confirmation.",
+   "fieldname": "welcome_url",
+   "fieldtype": "Data",
+   "label": "Welcome URL",
+   "options": "URL"
+  },
+  {
+   "default": "0",
+   "depends_on": "welcome_url",
+   "fieldname": "add_query_parameters",
+   "fieldtype": "Check",
+   "label": "Add Query Parameters"
   }
  ],
  "index_web_pages_for_search": 1,
@@ -51,10 +78,11 @@
    "link_fieldname": "email_group"
   }
  ],
- "modified": "2021-06-15 11:25:13.556201",
+ "modified": "2023-11-24 18:35:17.268492",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Group",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {
@@ -75,5 +103,6 @@
  "show_name_in_global_search": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/email/doctype/email_group/email_group.py

@@ -18,10 +18,12 @@ class EmailGroup(Document):
 	if TYPE_CHECKING:
 		from frappe.types import DF
 
+		add_query_parameters: DF.Check
 		confirmation_email_template: DF.Link | None
 		title: DF.Data
 		total_subscribers: DF.Int
 		welcome_email_template: DF.Link | None
+		welcome_url: DF.Data | None
 	# end: auto-generated types
 	def onload(self):
 		singles = [d.name for d in frappe.get_all("DocType", "name", {"issingle": 1})]
@@ -72,6 +74,22 @@ class EmailGroup(Document):
 			self.name,
 		)[0][0]
 
+	@frappe.whitelist()
+	def preview_welcome_url(self, email: str | None = None) -> str | None:
+		"""Get Welcome URL for the email group."""
+		return self.get_welcome_url(email)
+
+	def get_welcome_url(self, email: str | None = None) -> str | None:
+		"""Get Welcome URL for the email group."""
+		if not self.welcome_url:
+			return None
+
+		return (
+			add_query_params(self.welcome_url, {"email": email, "email_group": self.name})
+			if self.add_query_parameters
+			else self.welcome_url
+		)
+
 	def on_trash(self):
 		for d in frappe.get_all("Email Group Member", "name", {"email_group": self.name}):
 			frappe.delete_doc("Email Group Member", d.name)
@@ -124,3 +142,19 @@ def send_welcome_email(welcome_email, email, email_group):
 	args = dict(email=email, email_group=email_group)
 	message = frappe.render_template(welcome_email.response_, args)
 	frappe.sendmail(email, subject=welcome_email.subject, message=message)
+
+
+def add_query_params(url: str, params: dict) -> str:
+	from urllib.parse import urlencode, urlparse, urlunparse
+
+	if not params:
+		return url
+
+	query_string = urlencode(params)
+	parsed = list(urlparse(url))
+	if parsed[4]:
+		parsed[4] += f"&{query_string}"
+	else:
+		parsed[4] = query_string
+
+	return urlunparse(parsed)

frappe/email/doctype/email_group/test_email_group.py

@@ -1,9 +1,32 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
+import frappe
 from frappe.tests.utils import FrappeTestCase
+from frappe.utils import validate_url
 
 # test_records = frappe.get_test_records('Email Group')
 
 
 class TestEmailGroup(FrappeTestCase):
-	pass
+	def test_welcome_url(self):
+		email_group = frappe.new_doc("Email Group")
+		email_group.title = "Test"
+		email_group.welcome_url = "http://example.com/welcome?hello=world"
+		email_group.add_query_parameters = 1
+		email_group.insert()
+
+		welcome_url = email_group.get_welcome_url("mail@example.org")
+		self.assertTrue(validate_url(welcome_url))
+		self.assertIn(email_group.welcome_url, welcome_url)
+		self.assertIn("email_group=Test", welcome_url)
+		self.assertIn("email=mail%40example.org", welcome_url)
+
+		email_group.add_query_parameters = 0
+		welcome_url = email_group.get_welcome_url("mail@example.org")
+		self.assertTrue(validate_url(welcome_url))
+		self.assertIn(email_group.welcome_url, welcome_url)
+		self.assertNotIn("email_group=Test", welcome_url)
+		self.assertNotIn("email=mail%40example.org", welcome_url)
+
+		email_group.welcome_url = ""
+		self.assertEqual(email_group.get_welcome_url(), None)

frappe/email/doctype/email_template/email_template.json

@@ -52,12 +52,12 @@
    "fieldname": "response_html",
    "fieldtype": "Code",
    "label": "Response ",
-   "options": "HTML"
+   "options": "Jinja"
   }
  ],
  "icon": "fa fa-comment",
  "links": [],
- "modified": "2023-08-28 22:29:04.457992",
+ "modified": "2023-12-12 20:01:07.080625",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Template",

frappe/email/doctype/email_template/email_template.py

@@ -22,6 +22,7 @@ class EmailTemplate(Document):
 		subject: DF.Data
 		use_html: DF.Check
 	# end: auto-generated types
+
 	@property
 	def response_(self):
 		return self.response_html if self.use_html else self.response
@@ -48,7 +49,7 @@ class EmailTemplate(Document):
 
 @frappe.whitelist()
 def get_email_template(template_name, doc):
-	"""Returns the processed HTML of a email template with the given doc"""
+	"""Return the processed HTML of a email template with the given doc"""
 
 	email_template = frappe.get_doc("Email Template", template_name)
 	return email_template.get_formatted_email(doc)

frappe/email/doctype/newsletter/newsletter.py

@@ -359,19 +359,29 @@ def confirm_subscription(email, email_group=None):
 	if email_group is None:
 		email_group = get_default_email_group()
 
-	if not frappe.db.exists("Email Group", email_group):
-		frappe.get_doc({"doctype": "Email Group", "title": email_group}).insert(ignore_permissions=True)
+	try:
+		group = frappe.get_doc("Email Group", email_group)
+	except frappe.DoesNotExistError:
+		group = frappe.get_doc({"doctype": "Email Group", "title": email_group}).insert(
+			ignore_permissions=True
+		)
 
 	frappe.flags.ignore_permissions = True
 
 	add_subscribers(email_group, email)
 	frappe.db.commit()
 
-	frappe.respond_as_web_page(
-		_("Confirmed"),
-		_("{0} has been successfully added to the Email Group.").format(email),
-		indicator_color="green",
-	)
+	welcome_url = group.get_welcome_url(email)
+
+	if welcome_url:
+		frappe.local.response["type"] = "redirect"
+		frappe.local.response["location"] = welcome_url
+	else:
+		frappe.respond_as_web_page(
+			_("Confirmed"),
+			_("{0} has been successfully added to the Email Group.").format(email),
+			indicator_color="green",
+		)
 
 
 def get_list_context(context=None):

frappe/email/email_body.py

@@ -394,7 +394,7 @@ def get_email_html(template, args, subject, header=None, with_container=False):
 
 
 def inline_style_in_html(html):
-	"""Convert email.css and html to inline-styled html"""
+	"""Convert email.css and html to inline-styled html."""
 	from premailer import Premailer
 
 	from frappe.utils.jinja_globals import bundled_asset
@@ -460,7 +460,7 @@ def add_attachment(fname, fcontent, content_type=None, parent=None, content_id=N
 
 
 def get_message_id():
-	"""Returns Message ID created from doctype and name"""
+	"""Return Message ID created from doctype and name."""
 	return email.utils.make_msgid(domain=frappe.local.site)
 
 

frappe/email/receive.py

@@ -162,7 +162,7 @@ class EmailServer:
 		return
 
 	def get_messages(self, folder="INBOX"):
-		"""Returns new email messages."""
+		"""Return new email messages."""
 
 		self.latest_messages = []
 		self.seen_status = {}
@@ -864,7 +864,7 @@ class InboundMail(Email):
 
 	@staticmethod
 	def get_email_fields(doctype):
-		"""Returns Email related fields of a doctype."""
+		"""Return Email related fields of a doctype."""
 		fields = frappe._dict()
 
 		email_fields = ["subject_field", "sender_field", "sender_name_field"]

frappe/exceptions.py

@@ -64,7 +64,8 @@ class RequestToken(Exception):
 
 
 class Redirect(Exception):
-	http_status_code = 301
+	def __init__(self, http_status_code: int = 301):
+		self.http_status_code = http_status_code
 
 
 class CSRFTokenError(Exception):

frappe/frappeclient.py

@@ -109,7 +109,7 @@ class FrappeClient:
 	def get_list(
 		self, doctype, fields='["name"]', filters=None, limit_start=0, limit_page_length=None
 	):
-		"""Returns list of records of a particular type"""
+		"""Return list of records of a particular type."""
 		if not isinstance(fields, str):
 			fields = json.dumps(fields)
 		params = {
@@ -173,7 +173,7 @@ class FrappeClient:
 		return self.post_request({"cmd": "frappe.client.submit", "doc": frappe.as_json(doc)})
 
 	def get_value(self, doctype, fieldname=None, filters=None):
-		"""Returns a value form a document
+		"""Return a value from a document.
 
 		:param doctype: DocType to be queried
 		:param fieldname: Field to be returned (default `name`)
@@ -212,7 +212,7 @@ class FrappeClient:
 		return self.post_request({"cmd": "frappe.client.cancel", "doctype": doctype, "name": name})
 
 	def get_doc(self, doctype, name="", filters=None, fields=None):
-		"""Returns a single remote document
+		"""Return a single remote document.
 
 		:param doctype: DocType of the document to be returned
 		:param name: (optional) `name` of the document to be returned

frappe/geo/utils.py

@@ -19,7 +19,7 @@ def get_coords(doctype, filters, type):
 
 
 def convert_to_geojson(type, coords):
-	"""Converts GPS coordinates to geoJSON string."""
+	"""Convert GPS coordinates to geoJSON string."""
 	geojson = {"type": "FeatureCollection", "features": None}
 
 	if type == "location_field":
@@ -90,7 +90,7 @@ def return_coordinates(doctype, filters_sql):
 
 
 def get_coords_conditions(doctype, filters=None):
-	"""Returns SQL conditions with user permissions and filters for event queries."""
+	"""Return SQL conditions with user permissions and filters for event queries."""
 	from frappe.desk.reportview import get_filters_cond
 
 	if not frappe.has_permission(doctype):

frappe/installer.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-
+import configparser
+import gzip
 import json
 import os
 import re
@@ -52,7 +53,7 @@ def _new_site(
 ):
 	"""Install a new Frappe site"""
 
-	from frappe.utils import get_site_path, scheduler, touch_file
+	from frappe.utils import scheduler
 
 	if not force and os.path.exists(site):
 		print(f"Site {site} already exists")
@@ -419,7 +420,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
 
 	Note: All record linked linked to Module Def are also deleted.
 
-	Returns: list of deleted doctypes."""
+	Return: list of deleted doctypes."""
 	drop_doctypes = []
 
 	doctype_link_field_map = _get_module_linked_doctype_field_map()
@@ -449,7 +450,6 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
 def _delete_linked_documents(
 	module_name: str, doctype_linkfield_map: dict[str, str], dry_run: bool
 ) -> None:
-
 	"""Deleted all records linked with module def"""
 	for doctype, fieldname in doctype_linkfield_map.items():
 		for record in frappe.get_all(doctype, filters={fieldname: module_name}, pluck="name"):
@@ -461,7 +461,7 @@ def _delete_linked_documents(
 def _get_module_linked_doctype_field_map() -> dict[str, str]:
 	"""Get all the doctypes which have module linked with them.
 
-	returns ordered dictionary with doctype->link field mapping."""
+	Return ordered dictionary with doctype->link field mapping."""
 
 	# Hardcoded to change order of deletion
 	ordered_doctypes = [
@@ -664,32 +664,6 @@ def remove_missing_apps():
 				frappe.db.set_global("installed_apps", json.dumps(installed_apps))
 
 
-def extract_sql_from_archive(sql_file_path):
-	"""Return the path of an SQL file if the passed argument is the path of a gzipped
-	SQL file or an SQL file path. The path may be absolute or relative from the bench
-	root directory or the sites sub-directory.
-
-	Args:
-	        sql_file_path (str): Path of the SQL file
-
-	Returns:
-	        str: Path of the decompressed SQL file
-	"""
-	from frappe.utils import get_bench_relative_path
-
-	sql_file_path = get_bench_relative_path(sql_file_path)
-	# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
-	if sql_file_path.endswith("sql.gz"):
-		decompressed_file_name = extract_sql_gzip(sql_file_path)
-	else:
-		decompressed_file_name = sql_file_path
-
-	# convert archive sql to latest compatible
-	convert_archive_content(decompressed_file_name)
-
-	return decompressed_file_name
-
-
 def convert_archive_content(sql_file_path):
 	if frappe.conf.db_type == "mariadb":
 		# ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed
@@ -723,20 +697,6 @@ def convert_archive_content(sql_file_path):
 		old_sql_file_path.unlink()
 
 
-def extract_sql_gzip(sql_gz_path):
-	import subprocess
-
-	try:
-		original_file = sql_gz_path
-		decompressed_file = original_file.rstrip(".gz")
-		cmd = f"gzip --decompress --force < {original_file} > {decompressed_file}"
-		subprocess.check_call(cmd, shell=True)
-	except Exception:
-		raise
-
-	return decompressed_file
-
-
 def _guess_mariadb_version() -> tuple[int] | None:
 	# Using command-line because we *might* not have a connection yet and this command is required
 	# in non-interactive mode.
@@ -793,53 +753,58 @@ def is_downgrade(sql_file_path, verbose=False):
 
 	from semantic_version import Version
 
-	head = "INSERT INTO `tabInstalled Application` VALUES"
+	backup_version = extract_version_from_dump(sql_file_path)
+	if backup_version is None:
+		# This is likely an older backup, so try to extract another way
+		header = get_db_dump_header(sql_file_path).split("\n")
+		if "Version" in header[0]:
+			backup_version = header[0].split(":")[-1].strip()
 
-	with open(sql_file_path) as f:
-		for line in f:
-			if head in line:
-				# 'line' (str) format: ('2056588823','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',1,'frappe','v10.1.71-74 (3c50d5e) (v10.x.x)','v10.x.x'),('855c640b8e','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',2,'your_custom_app','0.0.1','master')
-				line = line.strip().lstrip(head).rstrip(";").strip()
-				app_rows = frappe.safe_eval(line)
-				# check if iterable consists of tuples before trying to transform
-				apps_list = (
-					app_rows
-					if all(isinstance(app_row, (tuple, list, set)) for app_row in app_rows)
-					else (app_rows,)
-				)
-				# 'all_apps' (list) format: [('frappe', '12.x.x-develop ()', 'develop'), ('your_custom_app', '0.0.1', 'master')]
-				all_apps = [x[-3:] for x in apps_list]
+	# Assume it's not a downgrade if we can't determine backup version
+	if backup_version is None:
+		return False
 
-				for app in all_apps:
-					app_name = app[0]
-					app_version = app[1].split(" ", 1)[0]
+	current_version = Version(frappe.__version__)
+	downgrade = Version(backup_version) < current_version
 
-					if app_name == "frappe":
-						try:
-							current_version = Version(frappe.__version__)
-							backup_version = Version(app_version[1:] if app_version[0] == "v" else app_version)
-						except ValueError:
-							return False
+	if verbose and downgrade:
+		print(f"Your site will be downgraded from Frappe {current_version} to {backup_version}")
 
-						downgrade = backup_version > current_version
-
-						if verbose and downgrade:
-							print(f"Your site will be downgraded from Frappe {backup_version} to {current_version}")
-
-						return downgrade
+	return downgrade
 
 
-def is_partial(sql_file_path):
-	with open(sql_file_path) as f:
-		header = " ".join(f.readline() for _ in range(5))
-		if "Partial Backup" in header:
-			return True
-	return False
+def extract_version_from_dump(sql_file_path: str) -> str | None:
+	"""
+	Extract frappe version from DB dump
+
+	:param sql_file_path: The path to the dump file
+	:return: The frappe version used to create the backup
+	"""
+	header = get_db_dump_header(sql_file_path).split("\n")
+	metadata = ""
+	if "begin frappe metadata" in header[0]:
+		for line in header[1:]:
+			if "end frappe metadata" in line:
+				break
+			metadata += line.replace("--", "").strip() + "\n"
+		parser = configparser.ConfigParser()
+		parser.read_string(metadata)
+		return parser["frappe"]["version"]
+	return None
+
+
+def is_partial(sql_file_path: str) -> bool:
+	"""
+	Function to return whether the database dump is a partial backup or not
+
+	:param sql_file_path: path to the database dump file
+	:return: True if the database dump is a partial backup, False otherwise
+	"""
+	header = get_db_dump_header(sql_file_path)
+	return "Partial Backup" in header
 
 
 def partial_restore(sql_file_path, verbose=False):
-	sql_file = extract_sql_from_archive(sql_file_path)
-
 	if frappe.conf.db_type == "mariadb":
 		from frappe.database.mariadb.setup_db import import_db_from_sql
 	elif frappe.conf.db_type == "postgres":
@@ -853,43 +818,60 @@ def partial_restore(sql_file_path, verbose=False):
 			fg="yellow",
 		)
 		warnings.warn(warn)
+	else:
+		click.secho("Unsupported database type", fg="red")
+		return
 
-	import_db_from_sql(source_sql=sql_file, verbose=verbose)
-
-	# Removing temporarily created file
-	if sql_file != sql_file_path:
-		os.remove(sql_file)
+	import_db_from_sql(source_sql=sql_file_path, verbose=verbose)
 
 
-def validate_database_sql(path, _raise=True):
-	"""Check if file has contents and if DefaultValue table exists
+def validate_database_sql(path: str, _raise: bool = True) -> None:
+	"""Check if file has contents and if `__Auth` table exists
 
 	Args:
 	        path (str): Path of the decompressed SQL file
 	        _raise (bool, optional): Raise exception if invalid file. Defaults to True.
 	"""
-	empty_file = False
-	missing_table = True
 
-	error_message = ""
+	if path.endswith(".gz"):
+		executable_name = "zgrep"
+	else:
+		executable_name = "grep"
 
-	if not os.path.getsize(path):
+	if os.path.getsize(path):
+		if (executable := which(executable_name)) is None:
+			frappe.throw(
+				f"`{executable_name}` not found in PATH! This is required to take a backup.",
+				exc=frappe.ExecutableNotFound,
+			)
+		try:
+			frappe.utils.execute_in_shell(f"{executable} -m1 __Auth {path}", check_exit_code=True)
+			return
+		except Exception:
+			error_message = "Table `__Auth` not found in file."
+	else:
 		error_message = f"{path} is an empty file!"
-		empty_file = True
-
-	# dont bother checking if empty file
-	if not empty_file:
-		with open(path) as f:
-			for line in f:
-				if "tabDefaultValue" in line:
-					missing_table = False
-					break
-
-		if missing_table:
-			error_message = "Table `tabDefaultValue` not found in file."
 
 	if error_message:
 		click.secho(error_message, fg="red")
 
-	if _raise and (missing_table or empty_file):
+	if _raise:
 		raise frappe.InvalidDatabaseFile
+
+
+def get_db_dump_header(file_path: str, file_bytes: int = 256) -> str:
+	"""
+	Get the header of a database dump file
+
+	:param file_path: path to the database dump file
+	:param file_bytes: number of bytes to read from the file
+	:return: The first few bytes of the file as requested
+	"""
+
+	# Use `gzip` to open the file if the extension is `.gz`
+	if file_path.endswith(".gz"):
+		with gzip.open(file_path, "rb") as f:
+			return f.read(file_bytes).decode()
+
+	with open(file_path, "rb") as f:
+		return f.read(file_bytes).decode()

frappe/integrations/doctype/google_calendar/google_calendar.py

@@ -202,9 +202,7 @@ def sync(g_calendar=None):
 
 
 def get_google_calendar_object(g_calendar):
-	"""
-	Returns an object of Google Calendar along with Google Calendar doc.
-	"""
+	"""Return an object of Google Calendar along with Google Calendar doc."""
 	google_settings = frappe.get_doc("Google Settings")
 	account = frappe.get_doc("Google Calendar", g_calendar)
 
@@ -257,8 +255,8 @@ def check_google_calendar(account, google_calendar):
 
 
 def sync_events_from_google_calendar(g_calendar, method=None):
-	"""
-	Syncs Events from Google Calendar in Framework Calendar.
+	"""Sync Events from Google Calendar in Framework Calendar.
+
 	Google Calendar returns nextSyncToken when all the events in Google Calendar are fetched.
 	nextSyncToken is returned at the very last page
 	https://developers.google.com/calendar/v3/sync
@@ -685,12 +683,10 @@ def format_date_according_to_google_calendar(all_day, starts_on, ends_on=None):
 
 
 def parse_google_calendar_recurrence_rule(repeat_day_week_number, repeat_day_name):
-	"""
-	Returns (repeat_on) exact date for combination eg 4TH viz. 4th thursday of a month
-	"""
+	"""Return (repeat_on) exact date for combination eg 4TH viz. 4th thursday of a month."""
 	if repeat_day_week_number < 0:
 		# Consider a month with 5 weeks and event is to be repeated in last week of every month, google caledar considers
-		# a month has 4 weeks and hence itll return -1 for a month with 5 weeks.
+		# a month has 4 weeks and hence it'll return -1 for a month with 5 weeks.
 		repeat_day_week_number = 4
 
 	weekdays = get_weekdays()
@@ -714,9 +710,7 @@ def parse_google_calendar_recurrence_rule(repeat_day_week_number, repeat_day_nam
 
 
 def repeat_on_to_google_calendar_recurrence_rule(doc):
-	"""
-	Returns event (repeat_on) in Google Calendar format ie RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH
-	"""
+	"""Return event (repeat_on) in Google Calendar format ie RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH."""
 	recurrence = framework_frequencies.get(doc.repeat_on)
 	weekdays = get_weekdays()
 
@@ -732,8 +726,8 @@ def repeat_on_to_google_calendar_recurrence_rule(doc):
 
 
 def get_week_number(dt):
-	"""
-	Returns the week number of the month for the specified date.
+	"""Return the week number of the month for the specified date.
+
 	https://stackoverflow.com/questions/3806473/python-week-number-of-the-month/16804556
 	"""
 	from math import ceil
@@ -771,9 +765,7 @@ def get_conference_data(doc):
 
 
 def get_attendees(doc):
-	"""
-	Returns a list of dicts with attendee emails, if available in event_participants table
-	"""
+	"""Return a list of dicts with attendee emails, if available in event_participants table."""
 	attendees, email_not_found = [], []
 
 	for participant in doc.event_participants:

frappe/integrations/doctype/google_contacts/google_contacts.py

@@ -74,9 +74,7 @@ def authorize_access(g_contact, reauthorize=False, code=None):
 
 
 def get_google_contacts_object(g_contact):
-	"""
-	Returns an object of Google Calendar along with Google Calendar doc.
-	"""
+	"""Return an object of Google Calendar along with Google Calendar doc."""
 	account = frappe.get_doc("Google Contacts", g_contact)
 	oauth_obj = GoogleOAuth("contacts")
 

frappe/integrations/doctype/google_drive/google_drive.py

@@ -88,9 +88,7 @@ def authorize_access(reauthorize=False, code=None):
 
 
 def get_google_drive_object():
-	"""
-	Returns an object of Google Drive.
-	"""
+	"""Return an object of Google Drive."""
 	account = frappe.get_doc("Google Drive")
 	oauth_obj = GoogleOAuth("drive")
 

frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py

@@ -21,7 +21,7 @@ class OAuthProviderSettings(Document):
 
 
 def get_oauth_settings():
-	"""Returns oauth settings"""
+	"""Return OAuth settings."""
 	return frappe._dict(
 		{
 			"skip_authorization": frappe.db.get_single_value(

frappe/integrations/google_oauth.py

@@ -56,7 +56,7 @@ class GoogleOAuth:
 			frappe.throw(frappe._("Please update {} before continuing.").format(google_settings))
 
 	def authorize(self, oauth_code: str) -> dict[str, str | int]:
-		"""Returns a dict with access and refresh token.
+		"""Return a dict with access and refresh token.
 
 		:param oauth_code: code got back from google upon successful auhtorization
 		"""
@@ -99,7 +99,7 @@ class GoogleOAuth:
 		)
 
 	def get_authentication_url(self, state: dict[str, str]) -> dict[str, str]:
-		"""Returns google authentication url.
+		"""Return Google authentication url.
 
 		:param state: dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc)
 		"""
@@ -117,7 +117,7 @@ class GoogleOAuth:
 		}
 
 	def get_google_service_object(self, access_token: str, refresh_token: str):
-		"""Returns google service object"""
+		"""Return Google service object."""
 
 		credentials_dict = {
 			"token": access_token,

frappe/integrations/utils.py

@@ -10,7 +10,9 @@ from frappe import _
 from frappe.utils import get_request_session
 
 
-def make_request(method, url, auth=None, headers=None, data=None, json=None, params=None):
+def make_request(
+	method: str, url: str, auth=None, headers=None, data=None, json=None, params=None
+):
 	auth = auth or ""
 	data = data or {}
 	headers = headers or {}
@@ -31,23 +33,71 @@ def make_request(method, url, auth=None, headers=None, data=None, json=None, par
 		raise exc
 
 
-def make_get_request(url, **kwargs):
+def make_get_request(url: str, **kwargs):
+	"""Make a 'GET' HTTP request to the given `url` and return processed response.
+
+	You can optionally pass the below parameters:
+
+	* `headers`: Headers to be set in the request.
+	* `params`: Query parameters to be passed in the request.
+	* `auth`: Auth credentials.
+	"""
 	return make_request("GET", url, **kwargs)
 
 
-def make_post_request(url, **kwargs):
+def make_post_request(url: str, **kwargs):
+	"""Make a 'POST' HTTP request to the given `url` and return processed response.
+
+	You can optionally pass the below parameters:
+
+	* `headers`: Headers to be set in the request.
+	* `data`: Data to be passed in body of the request.
+	* `json`: JSON to be passed in the request.
+	* `params`: Query parameters to be passed in the request.
+	* `auth`: Auth credentials.
+	"""
 	return make_request("POST", url, **kwargs)
 
 
-def make_put_request(url, **kwargs):
+def make_put_request(url: str, **kwargs):
+	"""Make a 'PUT' HTTP request to the given `url` and return processed response.
+
+	You can optionally pass the below parameters:
+
+	* `headers`: Headers to be set in the request.
+	* `data`: Data to be passed in body of the request.
+	* `json`: JSON to be passed in the request.
+	* `params`: Query parameters to be passed in the request.
+	* `auth`: Auth credentials.
+	"""
 	return make_request("PUT", url, **kwargs)
 
 
-def make_patch_request(url, **kwargs):
+def make_patch_request(url: str, **kwargs):
+	"""Make a 'PATCH' HTTP request to the given `url` and return processed response.
+
+	You can optionally pass the below parameters:
+
+	* `headers`: Headers to be set in the request.
+	* `data`: Data to be passed in body of the request.
+	* `json`: JSON to be passed in the request.
+	* `params`: Query parameters to be passed in the request.
+	* `auth`: Auth credentials.
+	"""
 	return make_request("PATCH", url, **kwargs)
 
 
-def make_delete_request(url, **kwargs):
+def make_delete_request(url: str, **kwargs):
+	"""Make a 'DELETE' HTTP request to the given `url` and return processed response.
+
+	You can optionally pass the below parameters:
+
+	* `headers`: Headers to be set in the request.
+	* `data`: Data to be passed in body of the request.
+	* `json`: JSON to be passed in the request.
+	* `params`: Query parameters to be passed in the request.
+	* `auth`: Auth credentials.
+	"""
 	return make_request("DELETE", url, **kwargs)
 
 

frappe/migrate.py

@@ -151,7 +151,7 @@ class SiteMigration:
 				frappe.get_attr(fn)()
 
 	def required_services_running(self) -> bool:
-		"""Returns True if all required services are running. Returns False and prints
+		"""Return True if all required services are running. Return False and print
 		instructions to stdout when required services are not available.
 		"""
 		service_status = check_connection(redis_services=["redis_cache"])

frappe/model/__init__.py

@@ -194,6 +194,8 @@ def get_permitted_fields(
 	parenttype: str | None = None,
 	user: str | None = None,
 	permission_type: str | None = None,
+	*,
+	ignore_virtual=False,
 ) -> list[str]:
 	meta = frappe.get_meta(doctype)
 	valid_columns = meta.get_valid_columns()
@@ -209,7 +211,10 @@ def get_permitted_fields(
 		permission_type = "select" if frappe.only_has_select_perm(doctype, user=user) else "read"
 
 	if permitted_fields := meta.get_permitted_fieldnames(
-		parenttype=parenttype, user=user, permission_type=permission_type
+		parenttype=parenttype,
+		user=user,
+		permission_type=permission_type,
+		with_virtual_fields=not ignore_virtual,
 	):
 		if permission_type == "select":
 			return permitted_fields

frappe/model/base_document.py

@@ -55,9 +55,9 @@ DOCTYPES_FOR_DOCTYPE = {"DocType", *TABLE_DOCTYPES_FOR_DOCTYPE.values()}
 
 
 def get_controller(doctype):
-	"""
-	Returns the locally cached **class** object of the given DocType.
-	For `custom` type, returns `frappe.model.document.Document`.
+	"""Return the locally cached **class** object of the given DocType.
+
+	For `custom` type, return `frappe.model.document.Document`.
 
 	:param doctype: DocType name as string.
 	"""
@@ -146,9 +146,9 @@ class BaseDocument:
 		return get_permitted_fields(doctype=self.doctype, parenttype=getattr(self, "parenttype", None))
 
 	def __getstate__(self):
-		"""
+		"""Return a copy of `__dict__` excluding unpicklable values like `meta`.
+
 		Called when pickling.
-		Returns a copy of `__dict__` excluding unpicklable values like `meta`.
 		More info: https://docs.python.org/3/library/pickle.html#handling-stateful-objects
 		"""
 
@@ -633,7 +633,7 @@ class BaseDocument:
 
 	def get_field_name_by_key_name(self, key_name):
 		"""MariaDB stores a mapping between `key_name` and `column_name`.
-		This function returns the `column_name` associated with the `key_name` passed
+		Return the `column_name` associated with the `key_name` passed.
 
 		Args:
 		        key_name (str): The name of the database index.
@@ -641,7 +641,7 @@ class BaseDocument:
 		Raises:
 		        IndexError: If the key is not found in the table.
 
-		Returns:
+		Return:
 		        str: The column name associated with the key.
 		"""
 		return frappe.db.sql(
@@ -660,12 +660,12 @@ class BaseDocument:
 		)[0].get("Column_name")
 
 	def get_label_from_fieldname(self, fieldname):
-		"""Returns the associated label for fieldname
+		"""Return the associated label for fieldname.
 
 		Args:
 		        fieldname (str): The fieldname in the DocType to use to pull the label.
 
-		Returns:
+		Return:
 		        str: The label associated with the fieldname, if found, otherwise `None`.
 		"""
 		df = self.meta.get_field(fieldname)
@@ -743,7 +743,7 @@ class BaseDocument:
 		return missing
 
 	def get_invalid_links(self, is_submittable=False):
-		"""Returns list of invalid links and also updates fetch values if not set"""
+		"""Return list of invalid links and also update fetch values if not set."""
 
 		def get_msg(df, docname):
 			# check if parentfield exists (only applicable for child table doctype)
@@ -1112,7 +1112,7 @@ class BaseDocument:
 		return "".join(set(pwd)) == "*"
 
 	def precision(self, fieldname, parentfield=None) -> int | None:
-		"""Returns float precision for a particular field (or get global default).
+		"""Return float precision for a particular field (or get global default).
 
 		:param fieldname: Fieldname for which precision is required.
 		:param parentfield: If fieldname is in child table."""
@@ -1174,7 +1174,7 @@ class BaseDocument:
 		return format_value(val, df=df, doc=doc, currency=currency, format=format)
 
 	def is_print_hide(self, fieldname, df=None, for_print=True):
-		"""Returns true if fieldname is to be hidden for print.
+		"""Return True if fieldname is to be hidden for print.
 
 		Print Hide can be set via the Print Format Builder or in the controller as a list
 		of hidden fields. Example
@@ -1203,7 +1203,8 @@ class BaseDocument:
 		return print_hide
 
 	def in_format_data(self, fieldname):
-		"""Returns True if shown via Print Format::`format_data` property.
+		"""Return True if shown via Print Format::`format_data` property.
+
 		Called from within standard print format."""
 		doc = getattr(self, "parent_doc", self)
 

frappe/model/db_query.py

@@ -335,7 +335,7 @@ class DatabaseQuery:
 		return args
 
 	def parse_args(self):
-		"""Convert fields and filters from strings to list, dicts"""
+		"""Convert fields and filters from strings to list, dicts."""
 		if isinstance(self.fields, str):
 			if self.fields == "*":
 				self.fields = ["*"]
@@ -635,6 +635,7 @@ class DatabaseQuery:
 			doctype=self.doctype,
 			parenttype=self.parent_doctype,
 			permission_type=self.permission_map.get(self.doctype),
+			ignore_virtual=True,
 		)
 
 		for i, field in enumerate(self.fields):
@@ -715,7 +716,8 @@ class DatabaseQuery:
 			j = j + len(permitted_fields) - 1
 
 	def prepare_filter_condition(self, f):
-		"""Returns a filter condition in the format:
+		"""Return a filter condition in the format:
+
 		ifnull(`tabDocType`.`fieldname`, fallback) operator "value"
 		"""
 
@@ -1079,6 +1081,8 @@ class DatabaseQuery:
 					self.fields[0].lower().startswith("count(")
 					or self.fields[0].lower().startswith("min(")
 					or self.fields[0].lower().startswith("max(")
+					or self.fields[0].lower().startswith("sum(")
+					or self.fields[0].lower().startswith("avg(")
 				)
 				and not self.group_by
 			)
@@ -1339,7 +1343,7 @@ def get_date_range(operator: str, value: str):
 
 
 def requires_owner_constraint(role_permissions):
-	"""Returns True if "select" or "read" isn't available without being creator."""
+	"""Return True if "select" or "read" isn't available without being creator."""
 
 	if not role_permissions.get("has_if_owner_enabled"):
 		return

frappe/model/delete_doc.py

@@ -254,7 +254,7 @@ def check_if_doc_is_linked(doc, method="Delete"):
 
 	for lf in link_fields:
 		link_dt, link_field, issingle = lf["parent"], lf["fieldname"], lf["issingle"]
-		if link_dt in ignored_doctypes or link_field == "amended_from":
+		if link_dt in ignored_doctypes or (link_field == "amended_from" and method == "Cancel"):
 			continue
 
 		try:

frappe/model/document.py

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
 
 
 def get_doc(*args, **kwargs):
-	"""returns a frappe.model.Document object.
+	"""Return a `frappe.model.Document` object.
 
 	:param arg1: Document dict or DocType name.
 	:param arg2: [optional] document name.
@@ -356,6 +356,7 @@ class Document(BaseDocument):
 			return self.insert()
 
 		self.check_if_locked()
+		self._set_defaults()
 		self.check_permission("write", "save")
 
 		self.set_user_and_timestamp()
@@ -454,7 +455,7 @@ class Document(BaseDocument):
 		return getattr(self, "_doc_before_save", None)
 
 	def has_value_changed(self, fieldname):
-		"""Returns true if value is changed before and after saving"""
+		"""Return True if value has changed before and after saving."""
 		previous = self.get_doc_before_save()
 		return previous.get(fieldname) != self.get(fieldname) if previous else True
 
@@ -771,16 +772,18 @@ class Document(BaseDocument):
 		if frappe.flags.in_import:
 			return
 
-		new_doc = frappe.new_doc(self.doctype, as_dict=True)
-		self.update_if_missing(new_doc)
+		if self.is_new():
+			new_doc = frappe.new_doc(self.doctype, as_dict=True)
+			self.update_if_missing(new_doc)
 
 		# children
 		for df in self.meta.get_table_fields():
-			new_doc = frappe.new_doc(df.options, as_dict=True)
+			new_doc = frappe.new_doc(df.options, parent_doc=self, parentfield=df.fieldname, as_dict=True)
 			value = self.get(df.fieldname)
 			if isinstance(value, list):
 				for d in value:
-					d.update_if_missing(new_doc)
+					if d.is_new():
+						d.update_if_missing(new_doc)
 
 	def check_if_latest(self):
 		"""Checks if `modified` timestamp provided by document being updated is same as the
@@ -924,7 +927,7 @@ class Document(BaseDocument):
 			frappe.throw(_("Cannot link cancelled document: {0}").format(msg), frappe.CancelledLinkError)
 
 	def get_all_children(self, parenttype=None) -> list["Document"]:
-		"""Returns all children documents from **Table** type fields in a list."""
+		"""Return all children documents from **Table** type fields in a list."""
 
 		children = []
 
@@ -977,7 +980,7 @@ class Document(BaseDocument):
 		if self.flags.notifications is None:
 
 			def _get_notifications():
-				"""returns enabled notifications for the current doctype"""
+				"""Return enabled notifications for the current doctype."""
 
 				return frappe.get_all(
 					"Notification",
@@ -1379,7 +1382,7 @@ class Document(BaseDocument):
 			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield"))))
 
 	def get_url(self):
-		"""Returns Desk URL for this document."""
+		"""Return Desk URL for this document."""
 		return get_absolute_url(self.doctype, self.name)
 
 	def add_comment(
@@ -1453,7 +1456,7 @@ class Document(BaseDocument):
 		)
 
 	def get_signature(self):
-		"""Returns signature (hash) for private URL."""
+		"""Return signature (hash) for private URL."""
 		return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest()
 
 	def get_document_share_key(self, expires_on=None, no_expiry=False):

frappe/model/mapper.py

@@ -10,8 +10,8 @@ from frappe.utils import cstr
 
 @frappe.whitelist()
 def make_mapped_doc(method, source_name, selected_children=None, args=None):
-	"""Returns the mapped document calling the given mapper method.
-	Sets selected_children as flags for the `get_mapped_doc` method.
+	"""Return the mapped document calling the given mapper method.
+	Set `selected_children` as flags for the `get_mapped_doc` method.
 
 	Called from `open_mapped_doc` from create_new.js"""
 
@@ -22,8 +22,7 @@ def make_mapped_doc(method, source_name, selected_children=None, args=None):
 
 	method = frappe.get_attr(method)
 
-	if method not in frappe.whitelisted:
-		raise frappe.PermissionError
+	frappe.is_whitelisted(method)
 
 	if selected_children:
 		selected_children = json.loads(selected_children)
@@ -38,15 +37,15 @@ def make_mapped_doc(method, source_name, selected_children=None, args=None):
 
 @frappe.whitelist()
 def map_docs(method, source_names, target_doc, args=None):
-	'''Returns the mapped document calling the given mapper method
-	with each of the given source docs on the target doc
+	"""Return the mapped document calling the given mapper method with each of the given source docs on the target doc.
 
 	:param args: Args as string to pass to the mapper method
-	E.g. args: "{ 'supplier': 'XYZ' }"'''
+
+	e.g. args: "{ 'supplier': 'XYZ' }"
+	"""
 
 	method = frappe.get_attr(method)
-	if method not in frappe.whitelisted:
-		raise frappe.PermissionError
+	frappe.is_whitelisted(method)
 
 	for src in json.loads(source_names):
 		_args = (src, target_doc, json.loads(args)) if args else (src, target_doc)

frappe/model/meta.py

@@ -208,7 +208,7 @@ class Meta(Document):
 		return self._table_fields
 
 	def get_global_search_fields(self):
-		"""Returns list of fields with `in_global_search` set and `name` if set"""
+		"""Return list of fields with `in_global_search` set and `name` if set."""
 		fields = self.get("fields", {"in_global_search": 1, "fieldtype": ["not in", no_value_fields]})
 		if getattr(self, "show_name_in_global_search", None):
 			fields.append(frappe._dict(fieldtype="Data", fieldname="name", label="Name"))
@@ -233,17 +233,17 @@ class Meta(Document):
 		return TABLE_DOCTYPES_FOR_DOCTYPE.get(fieldname)
 
 	def get_field(self, fieldname):
-		"""Return docfield from meta"""
+		"""Return docfield from meta."""
 
 		return self._fields.get(fieldname)
 
 	def has_field(self, fieldname):
-		"""Returns True if fieldname exists"""
+		"""Return True if fieldname exists."""
 
 		return fieldname in self._fields
 
 	def get_label(self, fieldname):
-		"""Get label of the given fieldname"""
+		"""Return label of the given fieldname."""
 		if df := self.get_field(fieldname):
 			return df.get("label")
 
@@ -273,8 +273,8 @@ class Meta(Document):
 		return search_fields
 
 	def get_fields_to_fetch(self, link_fieldname=None):
-		"""Returns a list of docfield objects for fields whose values
-		are to be fetched and updated for a particular link field
+		"""Return a list of docfield objects for fields whose values
+		are to be fetched and updated for a particular link field.
 
 		These fields are of type Data, Link, Text, Readonly and their
 		fetch_from property is set as `link_fieldname`.`source_fieldname`"""
@@ -565,7 +565,14 @@ class Meta(Document):
 			self.high_permlevel_fields = [df for df in self.fields if df.permlevel > 0]
 		return self.high_permlevel_fields
 
-	def get_permitted_fieldnames(self, parenttype=None, *, user=None, permission_type="read"):
+	def get_permitted_fieldnames(
+		self,
+		parenttype=None,
+		*,
+		user=None,
+		permission_type="read",
+		with_virtual_fields=True,
+	):
 		"""Build list of `fieldname` with read perm level and all the higher perm levels defined.
 
 		Note: If permissions are not defined for DocType, return all the fields with value.
@@ -590,7 +597,9 @@ class Meta(Document):
 
 		permitted_fieldnames.extend(
 			df.fieldname
-			for df in self.get_fieldnames_with_value(with_field_meta=True, with_virtual_fields=True)
+			for df in self.get_fieldnames_with_value(
+				with_field_meta=True, with_virtual_fields=with_virtual_fields
+			)
 			if df.permlevel in permlevel_access
 		)
 		return permitted_fieldnames
@@ -615,7 +624,7 @@ class Meta(Document):
 		return permissions
 
 	def get_dashboard_data(self):
-		"""Returns dashboard setup related to this doctype.
+		"""Return dashboard setup related to this doctype.
 
 		This method will return the `data` property in the `[doctype]_dashboard.py`
 		file in the doctype's folder, along with any overrides or extensions
@@ -677,15 +686,12 @@ class Meta(Document):
 					dict(label=link.group, items=[link.parent_doctype or link.link_doctype])
 				)
 
+			if not data.fieldname and link.link_fieldname:
+				data.fieldname = link.link_fieldname
+
 			if not link.is_child_table:
-				if link.link_fieldname != data.fieldname:
-					if data.fieldname:
-						data.non_standard_fieldnames[link.link_doctype] = link.link_fieldname
-					else:
-						data.fieldname = link.link_fieldname
+				data.non_standard_fieldnames[link.link_doctype] = link.link_fieldname
 			elif link.is_child_table:
-				if not data.fieldname:
-					data.fieldname = link.link_fieldname
 				data.internal_links[link.parent_doctype] = [link.table_fieldname, link.link_fieldname]
 
 	def get_row_template(self):
@@ -695,7 +701,7 @@ class Meta(Document):
 		return self.get_web_template(suffix="_list")
 
 	def get_web_template(self, suffix=""):
-		"""Returns the relative path of the row template for this doctype"""
+		"""Return the relative path of the row template for this doctype."""
 		module_name = frappe.scrub(self.module)
 		doctype = frappe.scrub(self.name)
 		template_path = frappe.get_module_path(

frappe/model/naming.py

@@ -334,7 +334,7 @@ def parse_naming_series(
 
 
 def has_custom_parser(e):
-	"""Returns true if the naming series part has a custom parser"""
+	"""Return True if the naming series part has a custom parser."""
 	return frappe.get_hooks("naming_series_variables", {}).get(e)
 
 

frappe/model/rename_doc.py

@@ -30,8 +30,7 @@ def update_document_title(
 	**kwargs,
 ) -> str:
 	"""
-	Update the name or title of a document. Returns `name` if document was renamed,
-	`docname` if renaming operation was queued.
+	Update the name or title of a document. Return `name` if document was renamed, `docname` if renaming operation was queued.
 
 	:param doctype: DocType of the document
 	:param docname: Name of the document

frappe/model/utils/__init__.py

@@ -84,7 +84,7 @@ def render_include(content):
 
 
 def get_fetch_values(doctype, fieldname, value):
-	"""Returns fetch value dict for the given object
+	"""Return fetch value dict for the given object.
 
 	:param doctype: Target doctype
 	:param fieldname: Link fieldname selected

frappe/modules/import_file.py

@@ -12,13 +12,10 @@ from frappe.utils import get_datetime, now
 
 
 def calculate_hash(path: str) -> str:
-	"""Calculate md5 hash of the file in binary mode
+	"""Calculate and return md5 hash of the file in binary mode.
 
 	Args:
 	        path (str): Path to the file to be hashed
-
-	Returns:
-	        str: The calculated hash
 	"""
 	hash_md5 = hashlib.md5(usedforsecurity=False)
 	with open(path, "rb") as f:
@@ -82,8 +79,8 @@ def import_file_by_path(
 	pre_process=None,
 	ignore_version: bool = None,
 	reset_permissions: bool = False,
-):
-	"""Import file from the given path
+) -> bool:
+	"""Import file from the given path.
 
 	Some conditions decide if a file should be imported or not.
 	Evaluation takes place in the order they are mentioned below.
@@ -107,8 +104,7 @@ def import_file_by_path(
 	        ignore_version (bool, optional): ignore current version. Defaults to None.
 	        reset_permissions (bool, optional): reset permissions for the file. Defaults to False.
 
-	Returns:
-	        [bool]: True if import takes place. False if it wasn't imported.
+	Return True if import takes place, False if it wasn't imported.
 	"""
 	try:
 		docs = read_doc_from_file(path)

frappe/modules/utils.py

@@ -22,10 +22,9 @@ doctype_python_modules = {}
 
 
 def export_module_json(doc: "Document", is_standard: bool, module: str) -> str | None:
-	"""Make a folder for the given doc and add its json file (make it a standard
-	object that will be synced)
+	"""Make a folder for the given doc and add its json file (make it a standard object that will be synced).
 
-	Returns the absolute file_path without the extension.
+	Return the absolute file_path without the extension.
 	Eg: For exporting a Print Format "_Test Print Format 1", the return value will be
 	`/home/gavin/frappe-bench/apps/frappe/frappe/core/print_format/_test_print_format_1/_test_print_format_1`
 	"""
@@ -181,12 +180,12 @@ def sync_customizations_for_doctype(data: dict, folder: str, filename: str = "")
 
 
 def scrub_dt_dn(dt: str, dn: str) -> tuple[str, str]:
-	"""Returns in lowercase and code friendly names of doctype and name for certain types"""
+	"""Return in lowercase and code friendly names of doctype and name for certain types."""
 	return scrub(dt), scrub(dn)
 
 
 def get_doc_path(module: str, doctype: str, name: str) -> str:
-	"""Returns path of a doc in a module"""
+	"""Return path of a doc in a module."""
 	return os.path.join(get_module_path(module), *scrub_dt_dn(doctype, name))
 
 
@@ -213,7 +212,7 @@ def export_doc(doctype, name, module=None):
 
 
 def get_doctype_module(doctype: str) -> str:
-	"""Returns **Module Def** name of given doctype."""
+	"""Return **Module Def** name of given doctype."""
 	doctype_module_map = frappe.cache.get_value(
 		"doctype_modules",
 		generator=lambda: dict(frappe.qb.from_("DocType").select("name", "module").run()),
@@ -226,7 +225,7 @@ def get_doctype_module(doctype: str) -> str:
 
 
 def load_doctype_module(doctype, module=None, prefix="", suffix=""):
-	"""Returns the module object for given doctype.
+	"""Return the module object for given doctype.
 
 	Note: This will return the standard defined module object for the doctype irrespective
 	of the `override_doctype_class` hook.

frappe/patches/v11_0/replicate_old_user_permissions.py

@@ -37,7 +37,7 @@ def execute():
 
 
 def get_doctypes_to_skip(doctype, user):
-	"""Returns doctypes to be skipped from user permission check"""
+	"""Return doctypes to be skipped from user permission check."""
 	doctypes_to_skip = []
 	valid_perms = get_user_valid_perms(user) or []
 	for perm in valid_perms:

frappe/patches/v13_0/queryreport_columns.py

@@ -7,7 +7,7 @@ import frappe
 
 
 def execute():
-	"""Convert Query Report json to support other content"""
+	"""Convert Query Report json to support other content."""
 	records = frappe.get_all("Report", filters={"json": ["!=", ""]}, fields=["name", "json"])
 	for record in records:
 		jstr = record["json"]

frappe/permissions.py

@@ -63,8 +63,8 @@ def has_permission(
 	*,
 	parent_doctype=None,
 ):
-	"""Returns True if user has permission `ptype` for given `doctype`.
-	If `doc` is passed, it also checks user, share and owner permissions.
+	"""Return True if user has permission `ptype` for given `doctype`.
+	If `doc` is passed, also check user, share and owner permissions.
 
 	:param doctype: DocType to check permission for
 	:param ptype: Permission Type to check
@@ -159,7 +159,7 @@ def has_permission(
 
 
 def get_doc_permissions(doc, user=None, ptype=None):
-	"""Returns a dict of evaluated permissions for given `doc` like `{"read":1, "write":1}`"""
+	"""Return a dict of evaluated permissions for given `doc` like `{"read":1, "write":1}`"""
 	if not user:
 		user = frappe.session.user
 
@@ -204,7 +204,7 @@ def get_doc_permissions(doc, user=None, ptype=None):
 
 def get_role_permissions(doctype_meta, user=None, is_owner=None):
 	"""
-	Returns dict of evaluated role permissions like
+	Return dict of evaluated role permissions like:
 	        {
 	                "read": 1,
 	                "write": 0,
@@ -272,7 +272,7 @@ def get_user_permissions(user):
 
 
 def has_user_permission(doc, user=None):
-	"""Returns True if User is allowed to view considering User Permissions"""
+	"""Return True if User is allowed to view considering User Permissions."""
 	from frappe.core.doctype.user_permission.user_permission import get_user_permissions
 
 	user_permissions = get_user_permissions(user)
@@ -374,7 +374,7 @@ def has_user_permission(doc, user=None):
 
 
 def has_controller_permissions(doc, ptype, user=None):
-	"""Returns controller permissions if defined. None if not defined"""
+	"""Return controller permissions if defined, None if not defined."""
 	if not user:
 		user = frappe.session.user
 
@@ -415,7 +415,7 @@ def get_valid_perms(doctype=None, user=None):
 
 
 def get_all_perms(role):
-	"""Returns valid permissions for a given role"""
+	"""Return valid permissions for a given role."""
 	perms = frappe.get_all("DocPerm", fields="*", filters=dict(role=role))
 	custom_perms = frappe.get_all("Custom DocPerm", fields="*", filters=dict(role=role))
 	doctypes_with_custom_perms = frappe.get_all("Custom DocPerm", pluck="parent", distinct=True)
@@ -462,7 +462,7 @@ def get_roles(user=None, with_standard=True):
 
 
 def get_doctype_roles(doctype, access_type="read"):
-	"""Returns a list of roles that are allowed to access passed doctype."""
+	"""Return a list of roles that are allowed to access the given `doctype`."""
 	meta = frappe.get_meta(doctype)
 	return [d.role for d in meta.get("permissions") if d.get(access_type)]
 
@@ -474,7 +474,7 @@ def get_perms_for(roles, perm_doctype="DocPerm"):
 
 
 def get_doctypes_with_custom_docperms():
-	"""Returns all the doctypes with Custom Docperms"""
+	"""Return all the doctypes with Custom Docperms."""
 
 	doctypes = frappe.get_all("Custom DocPerm", fields=["parent"], distinct=1)
 	return [d.parent for d in doctypes]
@@ -655,22 +655,18 @@ def get_doc_name(doc):
 
 
 def allow_everything():
-	"""
-	returns a dict with access to everything
-	eg. {"read": 1, "write": 1, ...}
-	"""
+	"""Return a dict with access to everything, eg. {"read": 1, "write": 1, ...}."""
 	return {ptype: 1 for ptype in rights}
 
 
 def get_allowed_docs_for_doctype(user_permissions, doctype):
-	"""Returns all the docs from the passed user_permissions that are
-	allowed under provided doctype"""
+	"""Return all the docs from the passed `user_permissions` that are allowed under provided doctype."""
 	return filter_allowed_docs_for_doctype(user_permissions, doctype, with_default_doc=False)
 
 
 def filter_allowed_docs_for_doctype(user_permissions, doctype, with_default_doc=True):
-	"""Returns all the docs from the passed user_permissions that are
-	allowed under provided doctype along with default doc value if with_default_doc is set"""
+	"""Return all the docs from the passed `user_permissions` that are
+	allowed under provided doctype along with default doc value if `with_default_doc` is set."""
 	allowed_doc = []
 	default_doc = None
 	for doc in user_permissions:

frappe/printing/doctype/print_format/print_format.json

@@ -112,14 +112,15 @@
    "label": "HTML",
    "oldfieldname": "html",
    "oldfieldtype": "Text Editor",
-   "options": "HTML"
+   "options": "Jinja"
   },
   {
    "depends_on": "raw_printing",
    "description": "Any string-based printer languages can be used. Writing raw commands requires knowledge of the printer's native language provided by the printer manufacturer. Please refer to the developer manual provided by the printer manufacturer on how to write their native commands. These commands are rendered on the server side using the Jinja Templating Language.",
    "fieldname": "raw_commands",
    "fieldtype": "Code",
-   "label": "Raw Commands"
+   "label": "Raw Commands",
+   "options": "Jinja"
   },
   {
    "depends_on": "eval:!doc.custom_format",
@@ -259,7 +260,7 @@
  "icon": "fa fa-print",
  "idx": 1,
  "links": [],
- "modified": "2023-08-28 20:25:09.660073",
+ "modified": "2023-12-12 19:59:37.133301",
  "modified_by": "Administrator",
  "module": "Printing",
  "name": "Print Format",

frappe/printing/doctype/print_format/print_format.py

@@ -48,6 +48,7 @@ class PrintFormat(Document):
 		show_section_headings: DF.Check
 		standard: DF.Literal["No", "Yes"]
 	# end: auto-generated types
+
 	def onload(self):
 		templates = frappe.get_all(
 			"Print Format Field Template",

frappe/public/js/form_builder/components/Field.vue

@@ -30,7 +30,7 @@ const label_input = ref(null);
 const hovered = ref(false);
 const selected = computed(() => store.selected(props.field.df.name));
 const component = computed(() => {
-	return props.field.df.fieldtype.replace(" ", "") + "Control";
+	return props.field.df.fieldtype.replaceAll(" ", "") + "Control";
 });
 
 function remove_field() {

frappe/public/js/form_builder/components/FieldProperties.vue

@@ -86,7 +86,7 @@ let docfield_df = computed(() => {
 		<div v-if="store.form.selected_field">
 			<div class="field" v-for="(df, i) in docfield_df" :key="i">
 				<component
-					:is="df.fieldtype.replace(' ', '') + 'Control'"
+					:is="df.fieldtype.replaceAll(' ', '') + 'Control'"
 					:args="args"
 					:df="df"
 					:read_only="store.read_only"

frappe/public/js/frappe/file_uploader/ImageCropper.vue

@@ -97,19 +97,19 @@ onMounted(() => {
 let aspect_ratio_buttons = computed(() => {
 	return [
 		{
-			label: __("1:1"),
+			label: __("1:1", null, "Image Cropper"),
 			value: 1,
 		},
 		{
-			label: __("4:3"),
+			label: __("4:3", null, "Image Cropper"),
 			value: 4 / 3,
 		},
 		{
-			label: __("16:9"),
+			label: __("16:9", null, "Image Cropper"),
 			value: 16 / 9,
 		},
 		{
-			label: __("Free"),
+			label: __("Free", null, "Image Cropper"),
 			value: NaN,
 		},
 	];

frappe/public/js/frappe/form/controls/attach.js

@@ -63,7 +63,6 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 	on_attach_doc_image() {
 		this.set_upload_options();
 		this.upload_options.restrictions.allowed_file_types = ["image/*"];
-		this.upload_options.restrictions.crop_image_aspect_ratio = 1;
 		this.file_uploader = new frappe.ui.FileUploader(this.upload_options);
 	}
 	set_upload_options() {

frappe/public/js/frappe/form/controls/code.js

@@ -160,6 +160,7 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex
 			JSON: "ace/mode/json",
 			Golang: "ace/mode/golang",
 			Go: "ace/mode/golang",
+			Jinja: "ace/mode/django",
 		};
 		const language = this.df.options;
 

frappe/public/js/frappe/form/form.js

@@ -346,13 +346,20 @@ frappe.ui.form.Form = class FrappeForm {
 
 		// using $.each to preserve df via closure
 		$.each(table_fields, function (i, df) {
-			frappe.model.on(df.options, "*", function (fieldname, value, doc) {
-				if (doc.parent == me.docname && doc.parentfield === df.fieldname) {
-					me.dirty();
-					me.fields_dict[df.fieldname].grid.set_value(fieldname, value, doc);
-					return me.script_manager.trigger(fieldname, doc.doctype, doc.name);
+			frappe.model.on(
+				df.options,
+				"*",
+				function (fieldname, value, doc, skip_dirty_trigger = false) {
+					if (doc.parent == me.docname && doc.parentfield === df.fieldname) {
+						if (!skip_dirty_trigger) {
+							me.dirty();
+						}
+
+						me.fields_dict[df.fieldname].grid.set_value(fieldname, value, doc);
+						return me.script_manager.trigger(fieldname, doc.doctype, doc.name);
+					}
 				}
-			});
+			);
 		});
 	}
 

frappe/public/js/frappe/form/sidebar/user_image.js

@@ -74,7 +74,7 @@ frappe.ui.form.setup_user_image_event = function (frm) {
 				}
 				field.$input.trigger("attach_doc_image");
 				// close sidebar
-				frm.page.close_sidebar();
+				frm.page.close_sidebar?.();
 			} else {
 				/// on remove event for a sidebar image wrapper remove attach file.
 				frm.attachments.remove_attachment_by_filename(

frappe/public/js/frappe/ui/capture.js

@@ -234,7 +234,7 @@ frappe.ui.Capture = class {
 
 	setup_remove_action() {
 		let me = this;
-		let elements = this.$template[0].getElementsByClassName("capture-remove-btn");
+		let elements = Array.from(this.$template[0].getElementsByClassName("capture-remove-btn"));
 
 		elements.forEach((el) => {
 			el.onclick = () => {

frappe/public/js/frappe/ui/field_group.js

@@ -17,7 +17,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 	}
 
 	make() {
-		var me = this;
+		let me = this;
 		if (this.fields) {
 			super.make();
 			this.refresh();
@@ -63,7 +63,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 	}
 
 	catch_enter_as_submit() {
-		var me = this;
+		let me = this;
 		$(this.body)
 			.find('input[type="text"], input[type="password"], select')
 			.keypress(function (e) {
@@ -77,7 +77,8 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 	}
 
 	get_input(fieldname) {
-		var field = this.fields_dict[fieldname];
+		let field = this.fields_dict[fieldname];
+		if (!field) return "";
 		return $(field.txt ? field.txt : field.input);
 	}
 
@@ -86,14 +87,14 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 	}
 
 	get_values(ignore_errors, check_invalid) {
-		var ret = {};
-		var errors = [];
+		let ret = {};
+		let errors = [];
 		let invalid = [];
 
-		for (var key in this.fields_dict) {
-			var f = this.fields_dict[key];
+		for (let key in this.fields_dict) {
+			let f = this.fields_dict[key];
 			if (f.get_value) {
-				var v = f.get_value();
+				let v = f.get_value();
 				if (f.df.reqd && is_null(typeof v === "string" ? strip_html(v) : v))
 					errors.push(__(f.df.label));
 
@@ -141,13 +142,13 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 	}
 
 	get_value(key) {
-		var f = this.fields_dict[key];
+		let f = this.fields_dict[key];
 		return f && (f.get_value ? f.get_value() : null);
 	}
 
 	set_value(key, val) {
 		return new Promise((resolve) => {
-			var f = this.fields_dict[key];
+			let f = this.fields_dict[key];
 			if (f) {
 				f.set_value(val).then(() => {
 					f.set_input?.(val);
@@ -170,7 +171,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 
 	set_values(dict) {
 		let promises = [];
-		for (var key in dict) {
+		for (let key in dict) {
 			if (this.fields_dict[key]) {
 				promises.push(this.set_value(key, dict[key]));
 			}
@@ -180,8 +181,8 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 	}
 
 	clear() {
-		for (var key in this.fields_dict) {
-			var f = this.fields_dict[key];
+		for (let key in this.fields_dict) {
+			let f = this.fields_dict[key];
 			if (f && f.set_input) {
 				f.set_input(f.df["default"] || "");
 			}