Merge remote-tracking branch 'upstream/develop' into po-translation

.git-blame-ignore-revs

@@ -40,3 +40,6 @@ f223bc02490902dfcc32892058f13f343d51fbaf
 
 # frappe.cache() -> frappe.cache
 fa6dc03cc87ad74e11609e7373078366fdcb3e1b
+
+# Bulk refactor with sourcery
+c35476256f85271fb57584eb0a26f4d9def3caf4

.github/workflows/labeller.yml

@@ -7,6 +7,6 @@ jobs:
   triage:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/linters.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
       - name: 'Setup Environment'
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
       - uses: actions/checkout@v4
@@ -57,7 +57,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
           cache: pip
@@ -76,7 +76,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 

.github/workflows/on_release.yml

@@ -24,7 +24,7 @@ jobs:
         with:
           node-version: 18
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Set up bench and build assets

.github/workflows/patch-mariadb-tests.yml

@@ -62,7 +62,7 @@ jobs:
           fi
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 

.github/workflows/publish-assets-develop.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 18
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
       - name: Set up bench and build assets

.github/workflows/server-tests.yml

@@ -83,7 +83,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.12'
 

.github/workflows/ui-tests.yml

@@ -65,7 +65,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.12'
 

cypress/integration/control_currency.js

@@ -9,6 +9,7 @@ context("Control Currency", () => {
 	function get_dialog_with_currency(df_options = {}) {
 		return cy.dialog({
 			title: "Currency Check",
+			animate: false,
 			fields: [
 				{
 					fieldname: fieldname,
@@ -76,6 +77,7 @@ context("Control Currency", () => {
 				});
 
 			get_dialog_with_currency(test_case.df_options).as("dialog");
+			cy.wait(300);
 			cy.get_field(fieldname, "Currency").clear();
 			cy.wait(300);
 			cy.fill_field(fieldname, test_case.input, "Currency").blur();

cypress/integration/control_float.js

@@ -7,6 +7,7 @@ context("Control Float", () => {
 	function get_dialog_with_float() {
 		return cy.dialog({
 			title: "Float Check",
+			animate: false,
 			fields: [
 				{
 					fieldname: "float_number",
@@ -19,6 +20,7 @@ context("Control Float", () => {
 
 	it("check value changes", () => {
 		get_dialog_with_float().as("dialog");
+		cy.wait(300);
 
 		let data = get_data();
 		data.forEach((x) => {

cypress/integration/control_phone.js

@@ -6,6 +6,10 @@ context("Control Phone", () => {
 		cy.visit("/app/website");
 	});
 
+	afterEach(() => {
+		cy.clear_dialogs();
+	});
+
 	function get_dialog_with_phone() {
 		return cy.dialog({
 			title: "Phone",
@@ -20,31 +24,37 @@ context("Control Phone", () => {
 
 	it("should set flag and data", () => {
 		get_dialog_with_phone().as("dialog");
+
 		cy.get(".selected-phone").click();
+		cy.wait(100);
 		cy.get(".phone-picker .phone-wrapper[id='afghanistan']").click();
+		cy.wait(100);
+		cy.get(".selected-phone .country").should("have.text", "+93");
+		cy.get(".selected-phone > img").should("have.attr", "src").and("include", "/af.svg");
+
 		cy.get(".selected-phone").click();
+		cy.wait(100);
 		cy.get(".phone-picker .phone-wrapper[id='india']").click();
+		cy.wait(100);
 		cy.get(".selected-phone .country").should("have.text", "+91");
 		cy.get(".selected-phone > img").should("have.attr", "src").and("include", "/in.svg");
 
 		let phone_number = "9312672712";
 		cy.get(".selected-phone > img").click().first();
-		cy.get_field("phone").first().click({ multiple: true });
+		cy.get_field("phone").first().click();
 		cy.get(".frappe-control[data-fieldname=phone]")
 			.findByRole("textbox")
 			.first()
-			.type(phone_number, { force: true });
+			.type(phone_number);
 
 		cy.get_field("phone").first().should("have.value", phone_number);
-		cy.get_field("phone").first().blur({ force: true });
+		cy.get_field("phone").first().blur();
 		cy.wait(100);
 		cy.get("@dialog").then((dialog) => {
 			let value = dialog.get_value("phone");
 			expect(value).to.equal("+91-" + phone_number);
 		});
-	});
 
-	it("case insensitive search for country and clear search", () => {
 		let search_text = "india";
 		cy.get(".selected-phone").click().first();
 		cy.get(".phone-picker").get(".search-phones").click().type(search_text);

frappe/__init__.py

@@ -109,7 +109,7 @@ class _dict(dict):
 
 
 def _(msg: str, lang: str | None = None, context: str | None = None) -> str:
-	"""Returns translated string in current lang, if exists.
+	"""Return translated string in current lang, if exists.
 	Usage:
 	        _('Change')
 	        _('Change', context='Coins')
@@ -148,8 +148,8 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str:
 	return translated_string or non_translated_string
 
 
-def as_unicode(text: str, encoding: str = "utf-8") -> str:
-	"""Convert to unicode if required"""
+def as_unicode(text, encoding: str = "utf-8") -> str:
+	"""Convert to unicode if required."""
 	if isinstance(text, str):
 		return text
 	elif text is None:
@@ -331,7 +331,7 @@ def connect_replica() -> bool:
 
 
 def get_site_config(sites_path: str | None = None, site_path: str | None = None) -> dict[str, Any]:
-	"""Returns `site_config.json` combined with `sites/common_site_config.json`.
+	"""Return `site_config.json` combined with `sites/common_site_config.json`.
 	`site_config` is a set of site wide settings like database name, password, email etc."""
 	config = _dict()
 
@@ -377,7 +377,7 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 
 
 def get_common_site_config(sites_path: str | None = None) -> dict[str, Any]:
-	"""Returns common site config as dictionary.
+	"""Return common site config as dictionary.
 
 	This is useful for:
 	- checking configuration which should only be allowed in common site config
@@ -436,7 +436,7 @@ def setup_redis_cache_connection():
 
 
 def get_traceback(with_context: bool = False) -> str:
-	"""Returns error traceback."""
+	"""Return error traceback."""
 	from frappe.utils import get_traceback
 
 	return get_traceback(with_context=with_context)
@@ -642,7 +642,7 @@ def get_user():
 
 
 def get_roles(username=None) -> list[str]:
-	"""Returns roles of current user."""
+	"""Return roles of current user."""
 	if not local.session or not local.session.user:
 		return ["Guest"]
 	import frappe.permissions
@@ -784,9 +784,9 @@ def sendmail(
 	return builder.process(send_now=now)
 
 
-whitelisted = []
-guest_methods = []
-xss_safe_methods = []
+whitelisted = set()
+guest_methods = set()
+xss_safe_methods = set()
 allowed_http_methods_for_whitelisted_func = {}
 
 
@@ -825,14 +825,14 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None):
 		else:
 			fn = validate_argument_types(fn, apply_condition=in_request_or_test)
 
-		whitelisted.append(fn)
+		whitelisted.add(fn)
 		allowed_http_methods_for_whitelisted_func[fn] = methods
 
 		if allow_guest:
-			guest_methods.append(fn)
+			guest_methods.add(fn)
 
 			if xss_safe:
-				xss_safe_methods.append(fn)
+				xss_safe_methods.add(fn)
 
 		return method or fn
 
@@ -1007,8 +1007,9 @@ def has_permission(
 	parent_doctype=None,
 ):
 	"""
-	Returns True if the user has permission `ptype` for given `doctype` or `doc`
-	Raises `frappe.PermissionError` if user isn't permitted and `throw` is truthy
+	Return True if the user has permission `ptype` for given `doctype` or `doc`.
+
+	Raise `frappe.PermissionError` if user isn't permitted and `throw` is truthy
 
 	:param doctype: DocType for which permission is to be check.
 	:param ptype: Permission type (`read`, `write`, `create`, `submit`, `cancel`, `amend`). Default: `read`.
@@ -1088,7 +1089,7 @@ def has_website_permission(doc=None, ptype="read", user=None, verbose=False, doc
 
 
 def is_table(doctype: str) -> bool:
-	"""Returns True if `istable` property (indicating child Table) is set for given DocType."""
+	"""Return True if `istable` property (indicating child Table) is set for given DocType."""
 
 	def get_tables():
 		return db.get_values("DocType", filters={"istable": 1}, order_by=None, pluck=True)
@@ -1132,7 +1133,7 @@ def new_doc(
 	as_dict: bool = False,
 	**kwargs,
 ) -> "Document":
-	"""Returns a new document of the given DocType with defaults set.
+	"""Return a new document of the given DocType with defaults set.
 
 	:param doctype: DocType of the new document.
 	:param parent_doc: [optional] add to parent document.
@@ -1156,6 +1157,7 @@ def set_value(doctype, docname, fieldname, value=None):
 
 
 def get_cached_doc(*args, **kwargs) -> "Document":
+	"""Identical to `frappe.get_doc`, but return from cache if available."""
 	if (key := can_cache_doc(args)) and (doc := cache.get_value(key)):
 		return doc
 
@@ -1178,7 +1180,7 @@ def _set_document_in_cache(key: str, doc: "Document") -> None:
 def can_cache_doc(args) -> str | None:
 	"""
 	Determine if document should be cached based on get_doc params.
-	Returns cache key if doc can be cached, None otherwise.
+	Return cache key if doc can be cached, None otherwise.
 	"""
 
 	if not args:
@@ -1430,17 +1432,17 @@ def rename_doc(
 
 
 def get_module(modulename):
-	"""Returns a module object for given Python module name using `importlib.import_module`."""
+	"""Return a module object for given Python module name using `importlib.import_module`."""
 	return importlib.import_module(modulename)
 
 
 def scrub(txt: str) -> str:
-	"""Returns sluggified string. e.g. `Sales Order` becomes `sales_order`."""
+	"""Return sluggified string. e.g. `Sales Order` becomes `sales_order`."""
 	return cstr(txt).replace(" ", "_").replace("-", "_").lower()
 
 
 def unscrub(txt: str) -> str:
-	"""Returns titlified string. e.g. `sales_order` becomes `Sales Order`."""
+	"""Return titlified string. e.g. `sales_order` becomes `Sales Order`."""
 	return txt.replace("_", " ").replace("-", " ").title()
 
 
@@ -1540,7 +1542,7 @@ def get_installed_apps(*, _ensure_on_bench=False) -> list[str]:
 
 
 def get_doc_hooks():
-	"""Returns hooked methods for given doc. It will expand the dict tuple if required."""
+	"""Return hooked methods for given doc. Expand the dict tuple if required."""
 	if not hasattr(local, "doc_events_hooks"):
 		hooks = get_hooks("doc_events", {})
 		out = {}
@@ -1647,7 +1649,7 @@ def setup_module_map():
 
 
 def get_file_items(path, raise_not_found=False, ignore_empty_lines=True):
-	"""Returns items from text file as a list. Ignores empty lines."""
+	"""Return items from text file as a list. Ignore empty lines."""
 	import frappe.utils
 
 	content = read_file(path, raise_not_found=raise_not_found)
@@ -2000,7 +2002,7 @@ def get_all(doctype, *args, **kwargs):
 
 
 def get_value(*args, **kwargs):
-	"""Returns a document property or list of properties.
+	"""Return a document property or list of properties.
 
 	Alias for `frappe.db.get_value`
 
@@ -2015,6 +2017,7 @@ def get_value(*args, **kwargs):
 
 
 def as_json(obj: dict | list, indent=1, separators=None, ensure_ascii=True) -> str:
+	"""Return the JSON string representation of the given `obj`."""
 	from frappe.utils.response import json_handler
 
 	if separators is None:
@@ -2047,7 +2050,7 @@ def are_emails_muted():
 
 
 def get_test_records(doctype):
-	"""Returns list of objects from `test_records.json` in the given doctype's folder."""
+	"""Return list of objects from `test_records.json` in the given doctype's folder."""
 	from frappe.modules import get_doctype_module, get_module_path
 
 	path = os.path.join(
@@ -2280,7 +2283,7 @@ log_level = None
 def logger(
 	module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20
 ):
-	"""Returns a python logger that uses StreamHandler"""
+	"""Return a python logger that uses StreamHandler."""
 	from frappe.utils.logger import get_logger
 
 	return get_logger(
@@ -2300,7 +2303,8 @@ def get_desk_link(doctype, name):
 	return html.format(doctype=doctype, name=name, doctype_local=_(doctype))
 
 
-def bold(text):
+def bold(text: str) -> str:
+	"""Return `text` wrapped in `<strong>` tags."""
 	return f"<strong>{text}</strong>"
 
 
@@ -2323,7 +2327,8 @@ def get_website_settings(key):
 	return local.website_settings.get(key)
 
 
-def get_system_settings(key):
+def get_system_settings(key: str):
+	"""Return the value associated with the given `key` from System Settings DocType."""
 	if not hasattr(local, "system_settings"):
 		try:
 			local.system_settings = get_cached_doc("System Settings")
@@ -2342,7 +2347,7 @@ def get_active_domains():
 
 def get_version(doctype, name, limit=None, head=False, raise_err=True):
 	"""
-	Returns a list of version information of a given DocType.
+	Return a list of version information for the given DocType.
 
 	Note: Applicable only if DocType has changes tracked.
 

frappe/auth.py

@@ -289,7 +289,7 @@ class LoginManager:
 	def check_password(self, user, pwd):
 		"""check password"""
 		try:
-			# returns user in correct case
+			# return user in correct case
 			return check_password(user, pwd)
 		except frappe.AuthenticationError:
 			self.fail("Incorrect password", user=user)

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -150,7 +150,7 @@ class AutoRepeat(Document):
 
 	def validate_auto_repeat_days(self):
 		auto_repeat_days = self.get_auto_repeat_days()
-		if not len(set(auto_repeat_days)) == len(auto_repeat_days):
+		if len(set(auto_repeat_days)) != len(auto_repeat_days):
 			repeated_days = get_repeated(auto_repeat_days)
 			plural = "s" if len(repeated_days) > 1 else ""
 
@@ -297,11 +297,11 @@ class AutoRepeat(Document):
 
 	def get_next_schedule_date(self, schedule_date, for_full_schedule=False):
 		"""
-		Returns the next schedule date for auto repeat after a recurring document has been created.
-		Adds required offset to the schedule_date param and returns the next schedule date.
+		Return the next schedule date for auto repeat after a recurring document has been created.
+		Add required offset to the schedule_date param and return the next schedule date.
 
 		:param schedule_date: The date when the last recurring document was created.
-		:param for_full_schedule: If True, returns the immediate next schedule date, else the full schedule.
+		:param for_full_schedule: If True, return the immediate next schedule date, else the full schedule.
 		"""
 		if month_map.get(self.frequency):
 			month_count = month_map.get(self.frequency) + month_diff(schedule_date, self.start_date) - 1

frappe/automation/doctype/milestone/milestone.json

@@ -53,7 +53,7 @@
  ],
  "in_create": 1,
  "links": [],
- "modified": "2022-08-03 12:20:55.076769",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Automation",
  "name": "Milestone",
@@ -74,7 +74,7 @@
  ],
  "quick_entry": 1,
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "title_field": "reference_type",
  "track_changes": 1

frappe/automation/doctype/milestone_tracker/milestone_tracker.json

@@ -35,7 +35,7 @@
   }
  ],
  "links": [],
- "modified": "2022-08-03 12:20:54.955953",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Automation",
  "name": "Milestone Tracker",
@@ -55,7 +55,7 @@
   }
  ],
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "track_changes": 1
 }

frappe/build.py

@@ -133,10 +133,9 @@ def setup_assets(assets_archive):
 	return directories_created
 
 
-def download_frappe_assets(verbose=True):
-	"""Downloads and sets up Frappe assets if they exist based on the current
-	commit HEAD.
-	Returns True if correctly setup else returns False.
+def download_frappe_assets(verbose=True) -> bool:
+	"""Download and set up Frappe assets if they exist based on the current commit HEAD.
+	Return True if correctly setup else return False.
 	"""
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
 
@@ -407,7 +406,7 @@ def link_assets_dir(source, target, hard_link=False):
 
 
 def scrub_html_template(content):
-	"""Returns HTML content with removed whitespace and comments"""
+	"""Return HTML content with removed whitespace and comments."""
 	# remove whitespace to a single space
 	content = WHITESPACE_PATTERN.sub(" ", content)
 
@@ -418,7 +417,7 @@ def scrub_html_template(content):
 
 
 def html_to_js_template(path, content):
-	"""returns HTML template content as Javascript code, adding it to `frappe.templates`"""
+	"""Return HTML template content as Javascript code, by adding it to `frappe.templates`."""
 	return """frappe.templates["{key}"] = '{content}';\n""".format(
 		key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content)
 	)

frappe/client.py

@@ -10,6 +10,7 @@ import frappe.utils
 from frappe import _
 from frappe.desk.reportview import validate_args
 from frappe.model.db_query import check_parent_permission
+from frappe.model.utils import is_virtual_doctype
 from frappe.utils import get_safe_filters
 from frappe.utils.deprecations import deprecated
 
@@ -37,7 +38,7 @@ def get_list(
 	as_dict: bool = True,
 	or_filters=None,
 ):
-	"""Returns a list of records by filters, fields, ordering and limit
+	"""Return a list of records by filters, fields, ordering and limit.
 
 	:param doctype: DocType of the data to be queried
 	:param fields: fields to be returned. Default is `name`
@@ -73,7 +74,7 @@ def get_count(doctype, filters=None, debug=False, cache=False):
 
 @frappe.whitelist()
 def get(doctype, name=None, filters=None, parent=None):
-	"""Returns a document by name or filters
+	"""Return a document by name or filters.
 
 	:param doctype: DocType of the document to be returned
 	:param name: return document of this `name`
@@ -96,7 +97,7 @@ def get(doctype, name=None, filters=None, parent=None):
 
 @frappe.whitelist()
 def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, parent=None):
-	"""Returns a value form a document
+	"""Return a value from a document.
 
 	:param doctype: DocType to be queried
 	:param fieldname: Field to be returned (default `name`)
@@ -295,7 +296,7 @@ def bulk_update(docs):
 
 @frappe.whitelist()
 def has_permission(doctype, docname, perm_type="read"):
-	"""Returns a JSON with data whether the document has the requested permission
+	"""Return a JSON with data whether the document has the requested permission.
 
 	:param doctype: DocType of the document to be checked
 	:param docname: `name` of the document to be checked
@@ -306,7 +307,7 @@ def has_permission(doctype, docname, perm_type="read"):
 
 @frappe.whitelist()
 def get_doc_permissions(doctype, docname):
-	"""Returns an evaluated document permissions dict like `{"read":1, "write":1}`
+	"""Return an evaluated document permissions dict like `{"read":1, "write":1}`.
 
 	:param doctype: DocType of the document to be evaluated
 	:param docname: `name` of the document to be evaluated
@@ -353,7 +354,7 @@ def get_js(items):
 
 @frappe.whitelist(allow_guest=True)
 def get_time_zone():
-	"""Returns default time zone"""
+	"""Return the default time zone."""
 	return {"time_zone": frappe.defaults.get_defaults().get("time_zone")}
 
 
@@ -431,6 +432,18 @@ def validate_link(doctype: str, docname: str, fields=None):
 		)
 
 	values = frappe._dict()
+
+	if is_virtual_doctype(doctype):
+		try:
+			frappe.get_doc(doctype, docname)
+			values.name = docname
+		except frappe.DoesNotExistError:
+			frappe.clear_last_message()
+			frappe.msgprint(
+				_("Document {0} {1} does not exist").format(frappe.bold(doctype), frappe.bold(docname)),
+			)
+		return values
+
 	values.name = frappe.db.get_value(doctype, docname, cache=True)
 
 	fields = frappe.parse_json(fields)
@@ -453,8 +466,7 @@ def validate_link(doctype: str, docname: str, fields=None):
 
 
 def insert_doc(doc) -> "Document":
-	"""Inserts document and returns parent document object with appended child document
-	if `doc` is child document else returns the inserted document object
+	"""Insert document and return parent document object with appended child document if `doc` is child document else return the inserted document object.
 
 	:param doc: doc to insert (dict)"""
 

frappe/commands/site.py

@@ -72,13 +72,10 @@ def new_site(
 	setup_db=True,
 ):
 	"Create a new site"
-	from frappe.installer import _new_site, extract_sql_from_archive
+	from frappe.installer import _new_site
 
 	frappe.init(site=site, new_site=True)
 
-	if source_sql:
-		source_sql = extract_sql_from_archive(source_sql)
-
 	_new_site(
 		db_name,
 		site,
@@ -180,75 +177,113 @@ def _restore(
 	with_public_files=None,
 	with_private_files=None,
 ):
+	from frappe.installer import extract_files
+	from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key
 
-	from frappe.installer import (
-		_new_site,
-		extract_files,
-		extract_sql_from_archive,
-		is_downgrade,
-		is_partial,
-		validate_database_sql,
-	)
-	from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key
+	err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True)
+	if err:
+		click.secho("Failed to detect type of backup file", fg="red")
+		sys.exit(1)
 
-	_backup = Backup(sql_file_path)
-
-	try:
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
-		if is_partial(decompressed_file_name):
-			click.secho(
-				"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
-				fg="red",
-			)
-			click.secho(
-				"Use `bench partial-restore` to restore a partial backup to an existing site.",
-				fg="yellow",
-			)
-			_backup.decryption_rollback()
-			sys.exit(1)
-
-	except UnicodeDecodeError:
-		_backup.decryption_rollback()
+	if "cipher" in out.decode().split(":")[-1].strip():
 		if encryption_key:
 			click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow")
-			_backup.backup_decryption(encryption_key)
 
 		else:
 			click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow")
 			encryption_key = get_or_generate_backup_encryption_key()
-			_backup.backup_decryption(encryption_key)
 
-		# Rollback on unsuccessful decryrption
-		if not os.path.exists(sql_file_path):
-			click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
+		with decrypt_backup(sql_file_path, encryption_key):
+			# Rollback on unsuccessful decryption
+			if not os.path.exists(sql_file_path):
+				click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
+				sys.exit(1)
 
-			_backup.decryption_rollback()
-			sys.exit(1)
-
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
-
-		if is_partial(decompressed_file_name):
-			click.secho(
-				"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
-				fg="red",
+			restore_backup(
+				sql_file_path,
+				site,
+				db_root_username,
+				db_root_password,
+				verbose,
+				install_app,
+				admin_password,
+				force,
 			)
-			click.secho(
-				"Use `bench partial-restore` to restore a partial backup to an existing site.",
-				fg="yellow",
-			)
-			_backup.decryption_rollback()
-			sys.exit(1)
+	else:
+		restore_backup(
+			sql_file_path,
+			site,
+			db_root_username,
+			db_root_password,
+			verbose,
+			install_app,
+			admin_password,
+			force,
+		)
 
-	validate_database_sql(decompressed_file_name, _raise=not force)
+	# Extract public and/or private files to the restored site, if user has given the path
+	if with_public_files:
+		# Decrypt data if there is a Key
+		if encryption_key:
+			with decrypt_backup(with_public_files, encryption_key):
+				public = extract_files(site, with_public_files)
+		else:
+			public = extract_files(site, with_public_files)
 
-	# dont allow downgrading to older versions of frappe without force
-	if not force and is_downgrade(decompressed_file_name, verbose=True):
+		# Removing temporarily created file
+		os.remove(public)
+
+	if with_private_files:
+		# Decrypt data if there is a Key
+		if encryption_key:
+			with decrypt_backup(with_private_files, encryption_key):
+				private = extract_files(site, with_private_files)
+		else:
+			private = extract_files(site, with_private_files)
+
+		# Removing temporarily created file
+		os.remove(private)
+
+	success_message = "Site {} has been restored{}".format(
+		site, " with files" if (with_public_files or with_private_files) else ""
+	)
+	click.secho(success_message, fg="green")
+
+
+def restore_backup(
+	sql_file_path: str,
+	site,
+	db_root_username,
+	db_root_password,
+	verbose,
+	install_app,
+	admin_password,
+	force,
+):
+	from frappe.installer import _new_site, is_downgrade, is_partial, validate_database_sql
+
+	if is_partial(sql_file_path):
+		click.secho(
+			"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
+			fg="red",
+		)
+		click.secho(
+			"Use `bench partial-restore` to restore a partial backup to an existing site.",
+			fg="yellow",
+		)
+		sys.exit(1)
+
+	# Check if the backup is of an older version of frappe and the user hasn't specified force
+	if is_downgrade(sql_file_path, verbose=True) and not force:
 		warn_message = (
 			"This is not recommended and may lead to unexpected behaviour. "
 			"Do you want to continue anyway?"
 		)
 		click.confirm(warn_message, abort=True)
 
+	# Validate the sql file
+	validate_database_sql(sql_file_path, _raise=not force)
+
 	try:
 		_new_site(
 			frappe.conf.db_name,
@@ -258,53 +293,15 @@ def _restore(
 			admin_password=admin_password,
 			verbose=verbose,
 			install_apps=install_app,
-			source_sql=decompressed_file_name,
+			source_sql=sql_file_path,
 			force=True,
 			db_type=frappe.conf.db_type,
 		)
 
 	except Exception as err:
 		print(err.args[1])
-		_backup.decryption_rollback()
 		sys.exit(1)
 
-	# Removing temporarily created file
-	if decompressed_file_name != sql_file_path:
-		os.remove(decompressed_file_name)
-		_backup.decryption_rollback()
-
-	# Extract public and/or private files to the restored site, if user has given the path
-	if with_public_files:
-		# Decrypt data if there is a Key
-		if encryption_key:
-			_backup = Backup(with_public_files)
-			_backup.backup_decryption(encryption_key)
-			if not os.path.exists(with_public_files):
-				_backup.decryption_rollback()
-		public = extract_files(site, with_public_files)
-
-		# Removing temporarily created file
-		os.remove(public)
-		_backup.decryption_rollback()
-
-	if with_private_files:
-		# Decrypt data if there is a Key
-		if encryption_key:
-			_backup = Backup(with_private_files)
-			_backup.backup_decryption(encryption_key)
-			if not os.path.exists(with_private_files):
-				_backup.decryption_rollback()
-		private = extract_files(site, with_private_files)
-
-		# Removing temporarily created file
-		os.remove(private)
-		_backup.decryption_rollback()
-
-	success_message = "Site {} has been restored{}".format(
-		site, " with files" if (with_public_files or with_private_files) else ""
-	)
-	click.secho(success_message, fg="green")
-
 
 @click.command("partial-restore")
 @click.argument("sql-file-path")
@@ -312,38 +309,23 @@ def _restore(
 @click.option("--encryption-key", help="Backup encryption key")
 @pass_context
 def partial_restore(context, sql_file_path, verbose, encryption_key=None):
-	from frappe.installer import extract_sql_from_archive, partial_restore
-	from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key
+	from frappe.installer import is_partial, partial_restore
+	from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key
 
 	if not os.path.exists(sql_file_path):
 		print("Invalid path", sql_file_path)
 		sys.exit(1)
 
 	site = get_site(context)
-	frappe.init(site=site)
-
-	_backup = Backup(sql_file_path)
-
 	verbose = context.verbose or verbose
-
+	frappe.init(site=site)
 	frappe.connect(site=site)
-	try:
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
+	err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True)
+	if err:
+		click.secho("Failed to detect type of backup file", fg="red")
+		sys.exit(1)
 
-		with open(decompressed_file_name) as f:
-			header = " ".join(f.readline() for _ in range(5))
-
-			# Check for full backup file
-			if "Partial Backup" not in header:
-				click.secho(
-					"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
-					fg="red",
-				)
-				_backup.decryption_rollback()
-				sys.exit(1)
-
-	except UnicodeDecodeError:
-		_backup.decryption_rollback()
+	if "cipher" in out.decode().split(":")[-1].strip():
 		if encryption_key:
 			click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow")
 			key = encryption_key
@@ -352,35 +334,30 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None):
 			click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow")
 			key = get_or_generate_backup_encryption_key()
 
-		_backup.backup_decryption(key)
-
-		# Rollback on unsuccessful decryrption
-		if not os.path.exists(sql_file_path):
-			click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
-			_backup.decryption_rollback()
-			sys.exit(1)
-
-		decompressed_file_name = extract_sql_from_archive(sql_file_path)
-
-		with open(decompressed_file_name) as f:
-			header = " ".join(f.readline() for _ in range(5))
-
-			# Check for Full backup file.
-			if "Partial Backup" not in header:
+		with decrypt_backup(sql_file_path, key):
+			if not is_partial(sql_file_path):
 				click.secho(
-					"Full Backup file detected.Use `bench restore` to restore a Frappe Site.",
+					"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
 					fg="red",
 				)
-				_backup.decryption_rollback()
 				sys.exit(1)
 
-	partial_restore(sql_file_path, verbose)
+			partial_restore(sql_file_path, verbose)
 
-	# Removing temporarily created file
-	_backup.decryption_rollback()
-	if os.path.exists(sql_file_path.rstrip(".gz")):
-		os.remove(sql_file_path.rstrip(".gz"))
+		# Rollback on unsuccessful decryption
+		if not os.path.exists(sql_file_path):
+			click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
+			sys.exit(1)
 
+	else:
+		if not is_partial(sql_file_path):
+			click.secho(
+				"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
+				fg="red",
+			)
+			sys.exit(1)
+
+		partial_restore(sql_file_path, verbose)
 	frappe.destroy()
 
 
@@ -524,6 +501,130 @@ def list_apps(context, format):
 		click.echo(frappe.as_json(summary_dict))
 
 
+@click.command("add-database-index")
+@click.option("--doctype", help="DocType on which index needs to be added")
+@click.option(
+	"--column",
+	multiple=True,
+	help="Column to index. Multiple columns will create multi-column index in given order. To create a multiple, single column index, execute the command multiple times.",
+)
+@pass_context
+def add_db_index(context, doctype, column):
+	"Adds a new DB index and creates a property setter to persist it."
+	from frappe.custom.doctype.property_setter.property_setter import make_property_setter
+
+	columns = column  # correct naming
+	for site in context.sites:
+		frappe.connect(site=site)
+		try:
+			frappe.db.add_index(doctype, columns)
+			if len(columns) == 1:
+				make_property_setter(
+					doctype,
+					columns[0],
+					property="search_index",
+					value="1",
+					property_type="Check",
+					for_doctype=False,  # Applied on docfield
+				)
+			frappe.db.commit()
+		finally:
+			frappe.destroy()
+
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+
+@click.command("describe-database-table")
+@click.option("--doctype", help="DocType to describe")
+@click.option(
+	"--column",
+	multiple=True,
+	help="Explicitly fetch accurate cardinality from table data. This can be quite slow on large tables.",
+)
+@pass_context
+def describe_database_table(context, doctype, column):
+	"""Describes various statistics about the table.
+
+	This is useful to build integration like
+	This includes:
+	1. Schema
+	2. Indexes
+	3. stats - total count of records
+	4. if column is specified then extra stats are generated for column:
+	        Distinct values count in column
+	"""
+	import json
+
+	for site in context.sites:
+		frappe.connect(site=site)
+		try:
+			data = _extract_table_stats(doctype, column)
+			# NOTE: Do not print anything else in this to avoid clobbering the output.
+			print(json.dumps(data, indent=2))
+		finally:
+			frappe.destroy()
+
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+
+def _extract_table_stats(doctype: str, columns: list[str]) -> dict:
+	from frappe.utils import cstr, get_table_name
+
+	def sql_bool(val):
+		return cstr(val).lower() in ("yes", "1", "true")
+
+	table = get_table_name(doctype, wrap_in_backticks=True)
+
+	schema = []
+	for field in frappe.db.sql(f"describe {table}", as_dict=True):
+		schema.append(
+			{
+				"column": field["Field"],
+				"type": field["Type"],
+				"is_nullable": sql_bool(field["Null"]),
+				"default": field["Default"],
+			}
+		)
+
+	def update_cardinality(column, value):
+		for col in schema:
+			if col["column"] == column:
+				col["cardinality"] = value
+				break
+
+	indexes = []
+	for idx in frappe.db.sql(f"show index from {table}", as_dict=True):
+		indexes.append(
+			{
+				"unique": not sql_bool(idx["Non_unique"]),
+				"cardinality": idx["Cardinality"],
+				"name": idx["Key_name"],
+				"sequence": idx["Seq_in_index"],
+				"nullable": sql_bool(idx["Null"]),
+				"column": idx["Column_name"],
+				"type": idx["Index_type"],
+			}
+		)
+		if idx["Seq_in_index"] == 1:
+			update_cardinality(idx["Column_name"], idx["Cardinality"])
+
+	total_rows = frappe.db.count(doctype)
+
+	# fetch accurate cardinality for columns by query. WARN: This can take a lot of time.
+	for column in columns:
+		cardinality = frappe.db.sql(f"select count(distinct {column}) from {table}")[0][0]
+		update_cardinality(column, cardinality)
+
+	return {
+		"table_name": table.strip("`"),
+		"total_rows": total_rows,
+		"schema": schema,
+		"indexes": indexes,
+	}
+
+
 @click.command("add-system-manager")
 @click.argument("email")
 @click.option("--first-name")
@@ -741,6 +842,9 @@ def use(site, sites_path="."):
 )
 @click.option("--verbose", default=False, is_flag=True, help="Add verbosity")
 @click.option("--compress", default=False, is_flag=True, help="Compress private and public files")
+@click.option(
+	"--old-backup-metadata", default=False, is_flag=True, help="Use older backup metadata"
+)
 @pass_context
 def backup(
 	context,
@@ -755,6 +859,7 @@ def backup(
 	compress=False,
 	include="",
 	exclude="",
+	old_backup_metadata=False,
 ):
 	"Backup"
 
@@ -780,6 +885,7 @@ def backup(
 				compress=compress,
 				verbose=verbose,
 				force=True,
+				old_backup_metadata=old_backup_metadata,
 			)
 		except Exception:
 			click.secho(
@@ -1289,7 +1395,7 @@ def trim_database(context, dry_run, format, no_backup, yes=False):
 		for table_name in database_tables:
 			if not table_name.startswith("tab"):
 				continue
-			if not (table_name.replace("tab", "", 1) in doctype_tables or table_name in STANDARD_TABLES):
+			if table_name.replace("tab", "", 1) not in doctype_tables and table_name not in STANDARD_TABLES:
 				TABLES_TO_DROP.append(table_name)
 
 		if not TABLES_TO_DROP:
@@ -1436,6 +1542,8 @@ def add_new_user(
 commands = [
 	add_system_manager,
 	add_user_for_sites,
+	add_db_index,
+	describe_database_table,
 	backup,
 	drop_site,
 	install_app,

frappe/contacts/doctype/address/address.py

@@ -143,7 +143,7 @@ def get_preferred_address(doctype, name, preferred_key="is_primary_address"):
 def get_default_address(
 	doctype: str, name: str | None, sort_key: str = "is_primary_address"
 ) -> str | None:
-	"""Returns default Address name for the given doctype, name"""
+	"""Return default Address name for the given doctype, name."""
 	if sort_key not in ["is_shipping_address", "is_primary_address"]:
 		return None
 
@@ -228,7 +228,7 @@ def get_address_list(doctype, txt, filters, limit_start, limit_page_length=20, o
 
 
 def has_website_permission(doc, ptype, user, verbose=False):
-	"""Returns true if there is a related lead or contact related to this document"""
+	"""Return True if there is a related lead or contact related to this document."""
 	contact_name = frappe.db.get_value("Contact", {"email_id": frappe.session.user})
 
 	if contact_name:

frappe/contacts/doctype/contact/contact.json

@@ -257,7 +257,7 @@
  "image_field": "image",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2023-10-02 12:00:27.299156",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Contacts",
  "name": "Contact",
@@ -392,7 +392,7 @@
  ],
  "show_title_field_in_link": 1,
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "title_field": "full_name"
 }

frappe/contacts/doctype/contact/contact.py

@@ -50,14 +50,14 @@ class Contact(Document):
 	def autoname(self):
 		self.name = self._get_full_name()
 
-		if frappe.db.exists("Contact", self.name):
-			self.name = append_number_if_name_exists("Contact", self.name)
-
 		# concat party name if reqd
 		for link in self.links:
 			self.name = self.name + "-" + link.link_name.strip()
 			break
 
+		if frappe.db.exists("Contact", self.name):
+			self.name = append_number_if_name_exists("Contact", self.name)
+
 	def validate(self):
 		self.full_name = self._get_full_name()
 		self.set_primary_email()
@@ -168,7 +168,7 @@ class Contact(Document):
 
 
 def get_default_contact(doctype, name):
-	"""Returns default contact for the given doctype, name"""
+	"""Return default contact for the given doctype, name."""
 	out = frappe.db.sql(
 		"""select parent,
 			IFNULL((select is_primary_contact from tabContact c where c.name = dl.parent), 0)

frappe/core/api/file.py

@@ -15,8 +15,7 @@ def unzip_file(name: str):
 
 @frappe.whitelist()
 def get_attached_images(doctype: str, names: list[str] | str) -> frappe._dict:
-	"""get list of image urls attached in form
-	returns {name: ['image.jpg', 'image.png']}"""
+	"""Return list of image urls attached in form `{name: ['image.jpg', 'image.png']}`."""
 
 	if isinstance(names, str):
 		names = json.loads(names)

frappe/core/doctype/communication/communication.py

@@ -298,7 +298,7 @@ class Communication(Document, CommunicationEmailMixin):
 
 	@staticmethod
 	def _get_emails_list(emails=None, exclude_displayname=False):
-		"""Returns list of emails from given email string.
+		"""Return list of emails from given email string.
 
 		* Removes duplicate mailids
 		* Removes display name from email address if exclude_displayname is True
@@ -309,15 +309,15 @@ class Communication(Document, CommunicationEmailMixin):
 		return [email.lower() for email in set(emails) if email]
 
 	def to_list(self, exclude_displayname=True):
-		"""Returns to list."""
+		"""Return `to` list."""
 		return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname)
 
 	def cc_list(self, exclude_displayname=True):
-		"""Returns cc list."""
+		"""Return `cc` list."""
 		return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname)
 
 	def bcc_list(self, exclude_displayname=True):
-		"""Returns bcc list."""
+		"""Return `bcc` list."""
 		return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname)
 
 	def get_attachments(self):
@@ -438,7 +438,7 @@ class Communication(Document, CommunicationEmailMixin):
 				frappe.db.commit()
 
 	def parse_email_for_timeline_links(self):
-		if not frappe.db.get_value("Email Account", self.email_account, "enable_automatic_linking"):
+		if not frappe.db.get_value("Email Account", filters={"enable_automatic_linking": 1}):
 			return
 
 		for doctype, docname in parse_email([self.recipients, self.cc, self.bcc]):
@@ -615,9 +615,9 @@ def parse_email(email_strings):
 
 
 def get_email_without_link(email):
-	"""
-	returns email address without doctype links
-	returns admin@example.com for email admin+doctype+docname@example.com
+	"""Return email address without doctype links.
+
+	e.g. 'admin@example.com' is returned for email 'admin+doctype+docname@example.com'
 	"""
 	if not frappe.get_all("Email Account", filters={"enable_automatic_linking": 1}):
 		return email
@@ -662,7 +662,10 @@ def update_parent_document_on_communication(doc):
 
 def update_first_response_time(parent, communication):
 	if parent.meta.has_field("first_response_time") and not parent.get("first_response_time"):
-		if is_system_user(communication.sender):
+		if (
+			is_system_user(communication.sender)
+			or frappe.get_cached_value("User", frappe.session.user, "user_type") == "System User"
+		):
 			if communication.sent_or_received == "Sent":
 				first_responded_on = communication.creation
 				if parent.meta.has_field("first_responded_on"):

frappe/core/doctype/communication/email.py

@@ -191,7 +191,8 @@ def _make(
 def validate_email(doc: "Communication") -> None:
 	"""Validate Email Addresses of Recipients and CC"""
 	if (
-		not (doc.communication_type == "Communication" and doc.communication_medium == "Email")
+		doc.communication_type != "Communication"
+		or doc.communication_medium != "Email"
 		or doc.flags.in_receive
 	):
 		return

frappe/core/doctype/communication/mixins.py

@@ -1,6 +1,9 @@
 import frappe
 from frappe import _
 from frappe.core.utils import get_parent_doc
+from frappe.desk.doctype.notification_settings.notification_settings import (
+	is_email_notifications_enabled_for_type,
+)
 from frappe.desk.doctype.todo.todo import ToDo
 from frappe.email.doctype.email_account.email_account import EmailAccount
 from frappe.utils import get_formatted_email, get_url, parse_addr
@@ -26,7 +29,7 @@ class CommunicationEmailMixin:
 		)
 
 	def get_email_with_displayname(self, email_address):
-		"""Returns email address after adding displayname."""
+		"""Return email address after adding displayname."""
 		display_name, email = parse_addr(email_address)
 		if display_name and display_name != email:
 			return email_address
@@ -78,7 +81,12 @@ class CommunicationEmailMixin:
 			if doc_owner := self.get_owner():
 				cc.append(doc_owner)
 			cc = set(cc) - {self.sender_mailid}
-			cc.update(self.get_assignees())
+			assignees = set(self.get_assignees())
+			# Check and remove If user disabled notifications for incoming emails on assigned document.
+			for assignee in assignees.copy():
+				if not is_email_notifications_enabled_for_type(assignee, "threads_on_assigned_document"):
+					assignees.remove(assignee)
+			cc.update(assignees)
 
 		cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc))
 		cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
@@ -143,7 +151,7 @@ class CommunicationEmailMixin:
 		return self.content
 
 	def get_attach_link(self, print_format):
-		"""Returns public link for the attachment via `templates/emails/print_link.html`."""
+		"""Return public link for the attachment via `templates/emails/print_link.html`."""
 		return frappe.get_template("templates/emails/print_link.html").render(
 			{
 				"url": get_url(),

frappe/core/doctype/data_import/data_import.js

@@ -136,47 +136,40 @@ frappe.ui.form.on("Data Import", {
 				let total_records = cint(r.message.total_records);
 
 				if (!total_records) return;
+				let action, message;
+				if (frm.doc.import_type === "Insert New Records") {
+					action = "imported";
+				} else {
+					action = "updated";
+				}
 
-				let message;
 				if (failed_records === 0) {
-					let message_args = [successful_records];
-					if (frm.doc.import_type === "Insert New Records") {
-						message =
-							successful_records > 1
-								? __("Successfully imported {0} records.", message_args)
-								: __("Successfully imported {0} record.", message_args);
+					let message_args = [action, successful_records];
+					if (successful_records === 1) {
+						message = __("Successfully {0} 1 record.", message_args);
 					} else {
-						message =
-							successful_records > 1
-								? __("Successfully updated {0} records.", message_args)
-								: __("Successfully updated {0} record.", message_args);
+						message = __("Successfully {0} {1} records.", message_args);
 					}
 				} else {
-					let message_args = [successful_records, total_records];
-					if (frm.doc.import_type === "Insert New Records") {
-						message =
-							successful_records > 1
-								? __(
-										"Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  )
-								: __(
-										"Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  );
+					let message_args = [action, successful_records, total_records];
+					if (successful_records === 1) {
+						message = __(
+							"Successfully {0} {1} record out of {2}. Click on Export Errored Rows, fix the errors and import again.",
+							message_args
+						);
 					} else {
-						message =
-							successful_records > 1
-								? __(
-										"Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  )
-								: __(
-										"Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-										message_args
-								  );
+						message = __(
+							"Successfully {0} {1} records out of {2}. Click on Export Errored Rows, fix the errors and import again.",
+							message_args
+						);
 					}
 				}
+
+				// If the job timed out, display an extra hint
+				if (r.message.status === "Timed Out") {
+					message += "<br/>" + __("Import timed out, please re-try.");
+				}
+
 				frm.dashboard.set_headline(message);
 			},
 		});

frappe/core/doctype/data_import/data_import.json

@@ -1,198 +1,198 @@
 {
-    "actions": [],
-    "autoname": "format:{reference_doctype} Import on {creation}",
-    "beta": 1,
-    "creation": "2019-08-04 14:16:08.318714",
-    "doctype": "DocType",
-    "editable_grid": 1,
-    "engine": "InnoDB",
-    "field_order": [
-        "reference_doctype",
-        "import_type",
-        "download_template",
-        "import_file",
-        "payload_count",
-        "html_5",
-        "google_sheets_url",
-        "refresh_google_sheet",
-        "column_break_5",
-        "status",
-        "submit_after_import",
-        "mute_emails",
-        "template_options",
-        "import_warnings_section",
-        "template_warnings",
-        "import_warnings",
-        "section_import_preview",
-        "import_preview",
-        "import_log_section",
-        "show_failed_logs",
-        "import_log_preview"
-    ],
-    "fields": [
-        {
-            "fieldname": "reference_doctype",
-            "fieldtype": "Link",
-            "in_list_view": 1,
-            "label": "Document Type",
-            "options": "DocType",
-            "reqd": 1,
-            "set_only_once": 1
-        },
-        {
-            "fieldname": "import_type",
-            "fieldtype": "Select",
-            "in_list_view": 1,
-            "label": "Import Type",
-            "options": "\nInsert New Records\nUpdate Existing Records",
-            "reqd": 1,
-            "set_only_once": 1
-        },
-        {
-            "depends_on": "eval:!doc.__islocal",
-            "fieldname": "import_file",
-            "fieldtype": "Attach",
-            "in_list_view": 1,
-            "label": "Import File",
-            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-        },
-        {
-            "fieldname": "import_preview",
-            "fieldtype": "HTML",
-            "label": "Import Preview"
-        },
-        {
-            "fieldname": "section_import_preview",
-            "fieldtype": "Section Break",
-            "label": "Preview"
-        },
-        {
-            "fieldname": "column_break_5",
-            "fieldtype": "Column Break"
-        },
-        {
-            "fieldname": "template_options",
-            "fieldtype": "Code",
-            "hidden": 1,
-            "label": "Template Options",
-            "options": "JSON",
-            "read_only": 1
-        },
-        {
-            "fieldname": "import_log_section",
-            "fieldtype": "Section Break",
-            "label": "Import Log"
-        },
-        {
-            "fieldname": "import_log_preview",
-            "fieldtype": "HTML",
-            "label": "Import Log Preview"
-        },
-        {
-            "default": "Pending",
-            "fieldname": "status",
-            "fieldtype": "Select",
-            "hidden": 1,
-            "label": "Status",
-            "no_copy": 1,
-            "options": "Pending\nSuccess\nPartial Success\nError",
-            "read_only": 1
-        },
-        {
-            "fieldname": "template_warnings",
-            "fieldtype": "Code",
-            "hidden": 1,
-            "label": "Template Warnings",
-            "options": "JSON"
-        },
-        {
-            "default": "0",
-            "fieldname": "submit_after_import",
-            "fieldtype": "Check",
-            "label": "Submit After Import",
-            "set_only_once": 1
-        },
-        {
-            "fieldname": "import_warnings_section",
-            "fieldtype": "Section Break",
-            "label": "Import File Errors and Warnings"
-        },
-        {
-            "fieldname": "import_warnings",
-            "fieldtype": "HTML",
-            "label": "Import Warnings"
-        },
-        {
-            "depends_on": "eval:!doc.__islocal",
-            "fieldname": "download_template",
-            "fieldtype": "Button",
-            "label": "Download Template"
-        },
-        {
-            "default": "1",
-            "fieldname": "mute_emails",
-            "fieldtype": "Check",
-            "label": "Don't Send Emails",
-            "set_only_once": 1
-        },
-        {
-            "default": "0",
-            "fieldname": "show_failed_logs",
-            "fieldtype": "Check",
-            "label": "Show Failed Logs"
-        },
-        {
-            "depends_on": "eval:!doc.__islocal && !doc.import_file",
-            "fieldname": "html_5",
-            "fieldtype": "HTML",
-            "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
-        },
-        {
-            "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
-            "description": "Must be a publicly accessible Google Sheets URL",
-            "fieldname": "google_sheets_url",
-            "fieldtype": "Data",
-            "label": "Import from Google Sheets",
-            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-        },
-        {
-            "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
-            "fieldname": "refresh_google_sheet",
-            "fieldtype": "Button",
-            "label": "Refresh Google Sheet"
-        },
-        {
-            "fieldname": "payload_count",
-            "fieldtype": "Int",
-            "hidden": 1,
-            "label": "Payload Count",
-            "read_only": 1
-        }
-    ],
-    "hide_toolbar": 1,
-    "links": [],
-    "modified": "2022-02-14 10:08:37.624914",
-    "modified_by": "Administrator",
-    "module": "Core",
-    "name": "Data Import",
-    "naming_rule": "Expression",
-    "owner": "Administrator",
-    "permissions": [
-        {
-            "create": 1,
-            "delete": 1,
-            "email": 1,
-            "export": 1,
-            "print": 1,
-            "read": 1,
-            "report": 1,
-            "role": "System Manager",
-            "share": 1,
-            "write": 1
-        }
-    ],
-    "sort_field": "modified",
-    "sort_order": "DESC",
-    "states": [],
-    "track_changes": 1
-}
+ "actions": [],
+ "autoname": "format:{reference_doctype} Import on {creation}",
+ "beta": 1,
+ "creation": "2019-08-04 14:16:08.318714",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "reference_doctype",
+  "import_type",
+  "download_template",
+  "import_file",
+  "payload_count",
+  "html_5",
+  "google_sheets_url",
+  "refresh_google_sheet",
+  "column_break_5",
+  "status",
+  "submit_after_import",
+  "mute_emails",
+  "template_options",
+  "import_warnings_section",
+  "template_warnings",
+  "import_warnings",
+  "section_import_preview",
+  "import_preview",
+  "import_log_section",
+  "show_failed_logs",
+  "import_log_preview"
+ ],
+ "fields": [
+  {
+   "fieldname": "reference_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Document Type",
+   "options": "DocType",
+   "reqd": 1,
+   "set_only_once": 1
+  },
+  {
+   "fieldname": "import_type",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Import Type",
+   "options": "\nInsert New Records\nUpdate Existing Records",
+   "reqd": 1,
+   "set_only_once": 1
+  },
+  {
+   "depends_on": "eval:!doc.__islocal",
+   "fieldname": "import_file",
+   "fieldtype": "Attach",
+   "in_list_view": 1,
+   "label": "Import File",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+  },
+  {
+   "fieldname": "import_preview",
+   "fieldtype": "HTML",
+   "label": "Import Preview"
+  },
+  {
+   "fieldname": "section_import_preview",
+   "fieldtype": "Section Break",
+   "label": "Preview"
+  },
+  {
+   "fieldname": "column_break_5",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "template_options",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "label": "Template Options",
+   "options": "JSON",
+   "read_only": 1
+  },
+  {
+   "fieldname": "import_log_section",
+   "fieldtype": "Section Break",
+   "label": "Import Log"
+  },
+  {
+   "fieldname": "import_log_preview",
+   "fieldtype": "HTML",
+   "label": "Import Log Preview"
+  },
+  {
+   "default": "Pending",
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "hidden": 1,
+   "label": "Status",
+   "no_copy": 1,
+   "options": "Pending\nSuccess\nPartial Success\nError\nTimed Out",
+   "read_only": 1
+  },
+  {
+   "fieldname": "template_warnings",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "label": "Template Warnings",
+   "options": "JSON"
+  },
+  {
+   "default": "0",
+   "fieldname": "submit_after_import",
+   "fieldtype": "Check",
+   "label": "Submit After Import",
+   "set_only_once": 1
+  },
+  {
+   "fieldname": "import_warnings_section",
+   "fieldtype": "Section Break",
+   "label": "Import File Errors and Warnings"
+  },
+  {
+   "fieldname": "import_warnings",
+   "fieldtype": "HTML",
+   "label": "Import Warnings"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal",
+   "fieldname": "download_template",
+   "fieldtype": "Button",
+   "label": "Download Template"
+  },
+  {
+   "default": "1",
+   "fieldname": "mute_emails",
+   "fieldtype": "Check",
+   "label": "Don't Send Emails",
+   "set_only_once": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "show_failed_logs",
+   "fieldtype": "Check",
+   "label": "Show Failed Logs"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && !doc.import_file",
+   "fieldname": "html_5",
+   "fieldtype": "HTML",
+   "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
+   "description": "Must be a publicly accessible Google Sheets URL",
+   "fieldname": "google_sheets_url",
+   "fieldtype": "Data",
+   "label": "Import from Google Sheets",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+  },
+  {
+   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
+   "fieldname": "refresh_google_sheet",
+   "fieldtype": "Button",
+   "label": "Refresh Google Sheet"
+  },
+  {
+   "fieldname": "payload_count",
+   "fieldtype": "Int",
+   "hidden": 1,
+   "label": "Payload Count",
+   "read_only": 1
+  }
+ ],
+ "hide_toolbar": 1,
+ "links": [],
+ "modified": "2023-12-15 12:45:49.452834",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Data Import",
+ "naming_rule": "Expression",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}

frappe/core/doctype/data_import/data_import.py

@@ -3,6 +3,8 @@
 
 import os
 
+from rq.timeouts import JobTimeoutException
+
 import frappe
 from frappe import _
 from frappe.core.doctype.data_import.exporter import Exporter
@@ -32,11 +34,13 @@ class DataImport(Document):
 		payload_count: DF.Int
 		reference_doctype: DF.Link
 		show_failed_logs: DF.Check
-		status: DF.Literal["Pending", "Success", "Partial Success", "Error"]
+		status: DF.Literal["Pending", "Success", "Partial Success", "Error", "Timed Out"]
 		submit_after_import: DF.Check
 		template_options: DF.Code | None
 		template_warnings: DF.Code | None
+
 	# end: auto-generated types
+
 	def validate(self):
 		doc_before_save = self.get_doc_before_save()
 		if (
@@ -136,6 +140,9 @@ def start_import(data_import):
 	try:
 		i = Importer(data_import.reference_doctype, data_import=data_import)
 		i.import_data()
+	except JobTimeoutException:
+		frappe.db.rollback()
+		data_import.db_set("status", "Timed Out")
 	except Exception:
 		frappe.db.rollback()
 		data_import.db_set("status", "Error")
@@ -190,6 +197,9 @@ def download_import_log(data_import_name):
 def get_import_status(data_import_name):
 	import_status = {}
 
+	data_import = frappe.get_doc("Data Import", data_import_name)
+	import_status["status"] = data_import.status
+
 	logs = frappe.get_all(
 		"Data Import Log",
 		fields=["count(*) as count", "success"],

frappe/core/doctype/data_import/data_import_list.js

@@ -20,13 +20,14 @@ frappe.listview_settings["Data Import"] = {
 			Success: "green",
 			"In Progress": "orange",
 			Error: "red",
+			"Timed Out": "orange",
 		};
 		let status = doc.status;
 
 		if (imports_in_progress.includes(doc.name)) {
 			status = "In Progress";
 		}
-		if (status == "Pending") {
+		if (status === "Pending") {
 			status = "Not Started";
 		}
 

frappe/core/doctype/data_import/importer.py

@@ -179,7 +179,7 @@ class Importer:
 
 					log_index += 1
 
-					if not self.data_import.status == "Partial Success":
+					if self.data_import.status != "Partial Success":
 						self.data_import.db_set("status", "Partial Success")
 
 					# commit after every successful import
@@ -514,8 +514,8 @@ class ImportFile:
 
 	def parse_next_row_for_import(self, data):
 		"""
-		Parses rows that make up a doc. A doc maybe built from a single row or multiple rows.
-		Returns the doc, rows, and data without the rows.
+		Parse rows that make up a doc. A doc maybe built from a single row or multiple rows.
+		Return the doc, rows, and data without the rows.
 		"""
 		doctypes = self.header.doctypes
 

frappe/core/doctype/deleted_document/deleted_document.py

@@ -27,6 +27,14 @@ class DeletedDocument(Document):
 	# end: auto-generated types
 	pass
 
+	@staticmethod
+	def clear_old_logs(days=180):
+		from frappe.query_builder import Interval
+		from frappe.query_builder.functions import Now
+
+		table = frappe.qb.DocType("Deleted Document")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
+
 
 @frappe.whitelist()
 def restore(name, alert=True):

frappe/core/doctype/docfield/docfield.py

@@ -118,9 +118,10 @@ class DocField(Document):
 		width: DF.Data | None
 	# end: auto-generated types
 	def get_link_doctype(self):
-		"""Returns the Link doctype for the docfield (if applicable)
-		if fieldtype is Link: Returns "options"
-		if fieldtype is Table MultiSelect: Returns "options" of the Link field in the Child Table
+		"""Return the Link doctype for the `docfield` (if applicable).
+
+		* If fieldtype is Link: Return "options".
+		* If fieldtype is Table MultiSelect: Return "options" of the Link field in the Child Table.
 		"""
 		if self.fieldtype == "Link":
 			return self.options

frappe/core/doctype/doctype/doctype.js

@@ -3,7 +3,7 @@
 
 frappe.ui.form.on("DocType", {
 	onload: function (frm) {
-		if (frm.is_new()) {
+		if (frm.is_new() && !frm.doc?.fields) {
 			frappe.listview_settings["DocType"].new_doctype_dialog();
 		}
 	},

frappe/core/doctype/doctype/doctype.json

@@ -68,6 +68,7 @@
   "column_break_51",
   "email_append_to",
   "sender_field",
+  "sender_name_field",
   "subject_field",
   "sb2",
   "permissions",
@@ -520,7 +521,7 @@
    "depends_on": "email_append_to",
    "fieldname": "sender_field",
    "fieldtype": "Data",
-   "label": "Sender Field",
+   "label": "Sender Email Field",
    "mandatory_depends_on": "email_append_to"
   },
   {
@@ -661,6 +662,12 @@
    "fieldtype": "Tab Break",
    "label": "Connections",
    "show_dashboard": 1
+  },
+  {
+   "depends_on": "email_append_to",
+   "fieldname": "sender_name_field",
+   "fieldtype": "Data",
+   "label": "Sender Name Field"
   }
  ],
  "icon": "fa fa-bolt",
@@ -743,7 +750,7 @@
    "link_fieldname": "reference_doctype"
   }
  ],
- "modified": "2023-11-01 16:45:14.960949",
+ "modified": "2023-12-01 18:37:16.799471",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",

frappe/core/doctype/doctype/doctype.py

@@ -162,6 +162,7 @@ class DocType(Document):
 		route: DF.Data | None
 		search_fields: DF.Data | None
 		sender_field: DF.Data | None
+		sender_name_field: DF.Data | None
 		show_name_in_global_search: DF.Check
 		show_preview_popup: DF.Check
 		show_title_field_in_link: DF.Check
@@ -177,6 +178,7 @@ class DocType(Document):
 		translated_doctype: DF.Check
 		website_search_field: DF.Data | None
 	# end: auto-generated types
+
 	def validate(self):
 		"""Validate DocType before saving.
 
@@ -290,7 +292,7 @@ class DocType(Document):
 		if not [d.fieldname for d in self.fields if d.in_list_view]:
 			cnt = 0
 			for d in self.fields:
-				if d.reqd and not d.hidden and not d.fieldtype in not_allowed_in_list_view:
+				if d.reqd and not d.hidden and d.fieldtype not in not_allowed_in_list_view:
 					d.in_list_view = 1
 					cnt += 1
 					if cnt == 4:
@@ -305,7 +307,7 @@ class DocType(Document):
 	def check_indexing_for_dashboard_links(self):
 		"""Enable indexing for outgoing links used in dashboard"""
 		for d in self.fields:
-			if d.fieldtype == "Link" and not (d.unique or d.search_index):
+			if d.fieldtype == "Link" and not d.unique and not d.search_index:
 				referred_as_link = frappe.db.exists(
 					"DocType Link",
 					{"parent": d.options, "link_doctype": self.name, "link_fieldname": d.fieldname},
@@ -412,7 +414,7 @@ class DocType(Document):
 
 		if self.has_web_view:
 			# route field must be present
-			if not "route" in [d.fieldname for d in self.fields]:
+			if "route" not in [d.fieldname for d in self.fields]:
 				frappe.throw(_('Field "route" is mandatory for Web Views'), title="Missing Field")
 
 			# clear website cache
@@ -984,7 +986,7 @@ class DocType(Document):
 		add_column(self.name, "parentfield", "Data")
 
 	def get_max_idx(self):
-		"""Returns the highest `idx`"""
+		"""Return the highest `idx`."""
 		max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", self.name)
 		return max_idx and max_idx[0][0] or 0
 
@@ -1265,7 +1267,7 @@ def validate_fields(meta):
 						),
 						WrongOptionsDoctypeLinkError,
 					)
-				elif not (options == d.options):
+				elif options != d.options:
 					frappe.throw(
 						_("{0}: Options {1} must be the same as doctype name {2} for the field {3}").format(
 							docname, d.options, options, d.label
@@ -1513,7 +1515,7 @@ def validate_fields(meta):
 
 	def check_table_multiselect_option(docfield):
 		"""check if the doctype provided in Option has atleast 1 Link field"""
-		if not docfield.fieldtype == "Table MultiSelect":
+		if docfield.fieldtype != "Table MultiSelect":
 			return
 
 		doctype = docfield.options
@@ -1579,7 +1581,7 @@ def validate_fields(meta):
 				title=_("Invalid Option"),
 			)
 
-		if not (meta.is_virtual == child_doctype_meta.is_virtual):
+		if meta.is_virtual != child_doctype_meta.is_virtual:
 			error_msg = " should be virtual." if meta.is_virtual else " cannot be virtual."
 			frappe.throw(
 				_("Child Table {0} for field {1}" + error_msg).format(
@@ -1666,22 +1668,12 @@ def validate_permissions_for_doctype(doctype, for_remove=False, alert=False):
 
 
 def clear_permissions_cache(doctype):
+	from frappe.cache_manager import clear_user_cache
+
 	frappe.clear_cache(doctype=doctype)
 	delete_notification_count_for(doctype)
-	for user in frappe.db.sql_list(
-		"""
-		SELECT
-			DISTINCT `tabHas Role`.`parent`
-		FROM
-			`tabHas Role`,
-			`tabDocPerm`
-		WHERE `tabDocPerm`.`parent` = %s
-			AND `tabDocPerm`.`role` = `tabHas Role`.`role`
-			AND `tabHas Role`.`parenttype` = 'User'
-		""",
-		doctype,
-	):
-		frappe.clear_cache(user=user)
+
+	clear_user_cache()
 
 
 def validate_permissions(doctype, for_remove=False, alert=False):
@@ -1891,7 +1883,7 @@ def check_email_append_to(doc):
 	if doc.sender_field and not sender_field:
 		frappe.throw(_("Select a valid Sender Field for creating documents from Email"))
 
-	if not sender_field.options == "Email":
+	if sender_field.options != "Email":
 		frappe.throw(_("Sender Field should have Email in options"))
 
 

frappe/core/doctype/doctype/test_doctype.py

@@ -786,6 +786,7 @@ def new_doctype(
 	depends_on: str = "",
 	fields: list[dict] | None = None,
 	custom: bool = True,
+	default: str | None = None,
 	**kwargs,
 ):
 	if not name:
@@ -803,6 +804,7 @@ def new_doctype(
 					"fieldname": "some_fieldname",
 					"fieldtype": "Data",
 					"unique": unique,
+					"default": default,
 					"depends_on": depends_on,
 				}
 			],

frappe/core/doctype/domain_settings/domain_settings.py

@@ -21,7 +21,7 @@ class DomainSettings(Document):
 		active_domains = [d.domain for d in self.active_domains]
 		added = False
 		for d in domains:
-			if not d in active_domains:
+			if d not in active_domains:
 				self.append("active_domains", dict(domain=d))
 				added = True
 

frappe/core/doctype/dynamic_link/dynamic_link.py

@@ -32,7 +32,7 @@ def deduplicate_dynamic_links(doc):
 	links, duplicate = [], False
 	for l in doc.links or []:
 		t = (l.link_doctype, l.link_name)
-		if not t in links:
+		if t not in links:
 			links.append(t)
 		else:
 			duplicate = True

frappe/core/doctype/error_log/error_log.json

@@ -70,7 +70,7 @@
  "idx": 1,
  "in_create": 1,
  "links": [],
- "modified": "2023-08-23 14:20:15.343339",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Error Log",
@@ -89,7 +89,7 @@
   }
  ],
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "title_field": "method"
 }

frappe/core/doctype/file/file.json

@@ -189,7 +189,7 @@
  "icon": "fa fa-file",
  "idx": 1,
  "links": [],
- "modified": "2023-08-02 09:43:51.178012",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "File",
@@ -217,7 +217,7 @@
   }
  ],
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "title_field": "file_name",
  "track_changes": 1

frappe/core/doctype/file/file.py

@@ -544,7 +544,7 @@ class File(Document):
 		return self._content
 
 	def get_full_path(self):
-		"""Returns file path from given file name"""
+		"""Return file path using the set file name."""
 
 		file_path = self.file_url or self.file_name
 
@@ -705,7 +705,7 @@ class File(Document):
 		return has_permission(self, "read")
 
 	def get_extension(self):
-		"""returns split filename and extension"""
+		"""Split and return filename and extension for the set `file_name`."""
 		return os.path.splitext(self.file_name)
 
 	def create_attachment_record(self):

frappe/core/doctype/module_def/module_def.json

@@ -123,7 +123,7 @@
    "link_fieldname": "module"
   }
  ],
- "modified": "2022-01-03 13:56:52.817954",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Module Def",
@@ -160,7 +160,7 @@
  ],
  "show_name_in_global_search": 1,
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "track_changes": 1
 }

frappe/core/doctype/module_def/module_def.py

@@ -47,7 +47,7 @@ class ModuleDef(Document):
 		if not frappe.local.module_app.get(frappe.scrub(self.name)):
 			with open(frappe.get_app_path(self.app_name, "modules.txt")) as f:
 				content = f.read()
-				if not self.name in content.splitlines():
+				if self.name not in content.splitlines():
 					modules = list(filter(None, content.splitlines()))
 					modules.append(self.name)
 

frappe/core/doctype/page/page.json

@@ -102,7 +102,7 @@
  "icon": "fa fa-file",
  "idx": 1,
  "links": [],
- "modified": "2023-10-22 22:41:25.568952",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Page",
@@ -129,7 +129,7 @@
   }
  ],
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "track_changes": 1
 }

frappe/core/doctype/page/page.py

@@ -118,7 +118,7 @@ class Page(Document):
 			shutil.rmtree(dir_path, ignore_errors=True)
 
 	def is_permitted(self):
-		"""Returns true if Has Role is not set or the user is allowed."""
+		"""Return True if `Has Role` is not set or the user is allowed."""
 		from frappe.utils import has_common
 
 		allowed = [

frappe/core/doctype/report/report.json

@@ -62,6 +62,8 @@
    "reqd": 1
   },
   {
+   "fetch_from": "ref_doctype.module",
+   "fetch_if_empty": 1,
    "fieldname": "module",
    "fieldtype": "Link",
    "label": "Module",

frappe/core/doctype/report/report.py

@@ -87,7 +87,8 @@ class Report(Document):
 		if (
 			self.is_standard == "Yes"
 			and not cint(getattr(frappe.local.conf, "developer_mode", 0))
-			and not (frappe.flags.in_migrate or frappe.flags.in_patch)
+			and not frappe.flags.in_migrate
+			and not frappe.flags.in_patch
 		):
 			frappe.throw(_("You are not allowed to delete Standard Report"))
 		delete_custom_role("report", self.name)
@@ -104,7 +105,7 @@ class Report(Document):
 				self.set("roles", roles)
 
 	def is_permitted(self):
-		"""Returns true if Has Role is not set or the user is allowed."""
+		"""Return True if `Has Role` is not set or the user is allowed."""
 		from frappe.utils import has_common
 
 		allowed = [
@@ -183,7 +184,7 @@ class Report(Document):
 	def execute_script(self, filters):
 		# server script
 		loc = {"filters": frappe._dict(filters), "data": None, "result": None}
-		safe_exec(self.report_script, None, loc)
+		safe_exec(self.report_script, None, loc, script_filename=f"Report {self.name}")
 		if loc["data"]:
 			return loc["data"]
 		else:

frappe/core/doctype/role/role.json

@@ -148,7 +148,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-08-05 18:33:27.694065",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Role",
@@ -169,7 +169,7 @@
  ],
  "quick_entry": 1,
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "track_changes": 1,
  "translated_doctype": 1

frappe/core/doctype/role/role.py

@@ -80,12 +80,23 @@ class Role(Document):
 		if frappe.flags.in_install:
 			return
 		if self.has_value_changed("desk_access"):
-			for user_name in get_users(self.name):
-				user = frappe.get_doc("User", user_name)
-				user_type = user.user_type
-				user.set_system_user()
-				if user_type != user.user_type:
-					user.save()
+			self.update_user_type_on_change()
+
+	def update_user_type_on_change(self):
+		"""When desk access changes, all the users that have this role need to be re-evaluated"""
+
+		users_with_role = get_users(self.name)
+
+		# perf: Do not re-evaluate users who already have same desk access that this role permits.
+		role_user_type = "System User" if self.desk_access else "Website User"
+		users_with_same_user_type = frappe.get_all("User", {"user_type": role_user_type}, pluck="name")
+
+		for user_name in set(users_with_role) - set(users_with_same_user_type):
+			user = frappe.get_doc("User", user_name)
+			user_type = user.user_type
+			user.set_system_user()
+			if user_type != user.user_type:
+				user.save()
 
 
 def get_info_based_on_role(role, field="email", ignore_permissions=False):

frappe/core/doctype/role_profile/role_profile.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
+from collections import defaultdict
+
 import frappe
 from frappe.model.document import Document
 
@@ -24,9 +26,24 @@ class RoleProfile(Document):
 
 	def on_update(self):
 		"""Changes in role_profile reflected across all its user"""
-		users = frappe.get_all("User", filters={"role_profile_name": self.name})
-		roles = [role.role for role in self.roles]
-		for d in users:
-			user = frappe.get_doc("User", d)
-			user.set("roles", [])
-			user.add_roles(*roles)
+		has_role = frappe.qb.DocType("Has Role")
+		user = frappe.qb.DocType("User")
+
+		all_current_roles = (
+			frappe.qb.from_(user)
+			.join(has_role)
+			.on(user.name == has_role.parent)
+			.where(user.role_profile_name == self.name)
+			.select(user.name, has_role.role)
+		).run()
+
+		user_roles = defaultdict(set)
+		for user, role in all_current_roles:
+			user_roles[user].add(role)
+
+		role_profile_roles = {role.role for role in self.roles}
+		for user, roles in user_roles.items():
+			if roles != role_profile_roles:
+				user = frappe.get_doc("User", user)
+				user.roles = []
+				user.add_roles(*role_profile_roles)

frappe/core/doctype/rq_job/rq_job.py

@@ -152,6 +152,12 @@ def serialize_job(job: Job) -> frappe._dict:
 	if matches := re.match(r"<function (?P<func_name>.*) at 0x.*>", job_name):
 		job_name = matches.group("func_name")
 
+	exc_info = None
+
+	# Get exc_string from the job result if it exists
+	if job_result := job.latest_result():
+		exc_info = job_result.exc_string
+
 	return frappe._dict(
 		name=job.id,
 		job_id=job.id,
@@ -161,7 +167,7 @@ def serialize_job(job: Job) -> frappe._dict:
 		started_at=convert_utc_to_system_timezone(job.started_at) if job.started_at else "",
 		ended_at=convert_utc_to_system_timezone(job.ended_at) if job.ended_at else "",
 		time_taken=(job.ended_at - job.started_at).total_seconds() if job.ended_at else "",
-		exc_info=job.exc_info,
+		exc_info=exc_info,
 		arguments=frappe.as_json(job.kwargs),
 		timeout=job.timeout,
 		creation=convert_utc_to_system_timezone(job.created_at),

frappe/core/doctype/server_script/server_script.py

@@ -128,14 +128,14 @@ class ServerScript(Document):
 			frappe.msgprint(str(e), title=_("Compilation warning"))
 
 	def execute_method(self) -> dict:
-		"""Specific to API endpoint Server Scripts
+		"""Specific to API endpoint Server Scripts.
 
-		Raises:
-		        frappe.DoesNotExistError: If self.script_type is not API
-		        frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user
+		Raise:
+		        frappe.DoesNotExistError: If self.script_type is not API.
+		        frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user.
 
-		Returns:
-		        dict: Evaluates self.script with frappe.utils.safe_exec.safe_exec and returns the flags set in it's safe globals
+		Return:
+		        dict: Evaluate self.script with frappe.utils.safe_exec.safe_exec and return the flags set in its safe globals.
 		"""
 
 		if self.enable_rate_limit:
@@ -155,7 +155,12 @@ class ServerScript(Document):
 		Args:
 		        doc (Document): Executes script with for a certain document's events
 		"""
-		safe_exec(self.script, _locals={"doc": doc}, restrict_commit_rollback=True)
+		safe_exec(
+			self.script,
+			_locals={"doc": doc},
+			restrict_commit_rollback=True,
+			script_filename=self.name,
+		)
 
 	def execute_scheduled_method(self):
 		"""Specific to Scheduled Jobs via Server Scripts
@@ -166,30 +171,28 @@ class ServerScript(Document):
 		if self.script_type != "Scheduler Event":
 			raise frappe.DoesNotExistError
 
-		safe_exec(self.script)
+		safe_exec(self.script, script_filename=self.name)
 
 	def get_permission_query_conditions(self, user: str) -> list[str]:
-		"""Specific to Permission Query Server Scripts
+		"""Specific to Permission Query Server Scripts.
 
 		Args:
-		        user (str): Takes user email to execute script and return list of conditions
+		        user (str): Take user email to execute script and return list of conditions.
 
-		Returns:
-		        list: Returns list of conditions defined by rules in self.script
+		Return:
+		        list: Return list of conditions defined by rules in self.script.
 		"""
 		locals = {"user": user, "conditions": ""}
-		safe_exec(self.script, None, locals)
+		safe_exec(self.script, None, locals, script_filename=self.name)
 		if locals["conditions"]:
 			return locals["conditions"]
 
 	@frappe.whitelist()
 	def get_autocompletion_items(self):
-		"""Generates a list of a autocompletion strings from the context dict
+		"""Generate a list of autocompletion strings from the context dict
 		that is used while executing a Server Script.
 
-		Returns:
-		        list: Returns list of autocompletion items.
-		        For e.g., ["frappe.utils.cint", "frappe.get_all", ...]
+		e.g., ["frappe.utils.cint", "frappe.get_all", ...]
 		"""
 
 		def get_keys(obj):
@@ -278,7 +281,7 @@ def execute_api_server_script(script=None, *args, **kwargs):
 		raise frappe.PermissionError
 
 	# output can be stored in flags
-	_globals, _locals = safe_exec(script.script)
+	_globals, _locals = safe_exec(script.script, script_filename=script.name)
 
 	return _globals.frappe.flags
 

frappe/core/doctype/server_script/server_script_utils.py

@@ -23,7 +23,7 @@ EVENT_MAP = {
 
 def run_server_script_for_doc_event(doc, event):
 	# run document event method
-	if not event in EVENT_MAP:
+	if event not in EVENT_MAP:
 		return
 
 	if frappe.flags.in_install:

frappe/core/doctype/sms_log/README.md

@@ -0,0 +1 @@
+Log of SMS sent via SMS Center.

frappe/core/doctype/sms_log/sms_log.js

@@ -0,0 +1,6 @@
+// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("SMS Log", {
+	refresh: function (frm) {},
+});

frappe/core/doctype/sms_log/sms_log.json

@@ -0,0 +1,371 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "autoname": "SYS-SMS-.#####", 
+ "beta": 0, 
+ "creation": "2012-03-27 14:36:47", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "editable_grid": 0, 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "sender_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Sender Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "sent_on", 
+   "fieldtype": "Date", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Sent On", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break0", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0, 
+   "width": "50%"
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "message", 
+   "fieldtype": "Small Text", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Message", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "sec_break1", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Simple", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "no_of_requested_sms", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "No of Requested SMS", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "requested_numbers", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Requested Numbers", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break1", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0, 
+   "width": "50%"
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "no_of_sent_sms", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "No of Sent SMS", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_in_quick_entry": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "sent_to", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Sent To", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "translatable": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "icon": "fa fa-mobile-phone", 
+ "idx": 1, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2018-08-21 16:15:40.898889", 
+ "modified_by": "Administrator", 
+ "module": "Core", 
+ "name": "SMS Log", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "cancel": 0, 
+   "create": 0, 
+   "delete": 0, 
+   "email": 1, 
+   "export": 0, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 0, 
+   "submit": 0, 
+   "write": 0
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "track_changes": 1, 
+ "track_seen": 0, 
+ "track_views": 0
+}

frappe/core/doctype/sms_log/sms_log.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: GNU General Public License v3. See license.txt
+
+
+from frappe.model.document import Document
+
+
+class SMSLog(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		message: DF.SmallText | None
+		no_of_requested_sms: DF.Int
+		no_of_sent_sms: DF.Int
+		requested_numbers: DF.Code | None
+		sender_name: DF.Data | None
+		sent_on: DF.Date | None
+		sent_to: DF.Code | None
+	# end: auto-generated types
+
+	pass

frappe/core/doctype/sms_log/test_sms_log.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+import unittest
+
+# test_records = frappe.get_test_records('SMS Log')
+
+
+class TestSMSLog(unittest.TestCase):
+	pass

frappe/core/doctype/sms_settings/sms_settings.py

@@ -46,7 +46,7 @@ def validate_receiver_nos(receiver_list):
 
 @frappe.whitelist()
 def get_contact_number(contact_name, ref_doctype, ref_name):
-	"returns mobile number of the contact"
+	"Return mobile number of the given contact."
 	number = frappe.db.sql(
 		"""select mobile_no, phone from tabContact
 		where name=%s

frappe/core/doctype/system_settings/system_settings.js

@@ -32,10 +32,25 @@ frappe.ui.form.on("System Settings", {
 			frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0);
 		}
 	},
-	on_update: function (frm) {
-		if (frappe.boot.time_zone && frappe.boot.time_zone.system !== frm.doc.time_zone) {
-			// Clear cache after saving to refresh the values of boot.
-			frappe.ui.toolbar.clear_cache();
+	after_save: function (frm) {
+		/**
+		 * Checks whether the effective value has changed.
+		 *
+		 * @param {Array.<string>} - Tuple with new fallback, previous fallback and
+		 *   optionally an override value.
+		 * @returns {boolean} - Whether the resulting value has effectively changed
+		 */
+		const has_effectively_changed = ([new_fallback, prev_fallback, override = undefined]) =>
+			!override && prev_fallback !== new_fallback;
+
+		const attr_tuples = [
+			[frm.doc.language, frappe.boot.sysdefaults.language, frappe.boot.user.language],
+			[frm.doc.rounding_method, frappe.boot.sysdefaults.rounding_method], // no user override.
+		];
+
+		if (attr_tuples.some(has_effectively_changed)) {
+			frappe.msgprint(__("Refreshing..."));
+			window.location.reload();
 		}
 	},
 	first_day_of_the_week(frm) {

frappe/core/doctype/system_settings/system_settings.json

@@ -639,7 +639,7 @@
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2023-11-27 14:08:01.927794",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",
@@ -655,7 +655,7 @@
  ],
  "quick_entry": 1,
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "track_changes": 1
 }

frappe/core/doctype/user/user.js

@@ -114,22 +114,6 @@ frappe.ui.form.on("User", {
 			return;
 		}
 
-		const hasChanged = (doc_attr, boot_attr) => {
-			return doc_attr && boot_attr && doc_attr !== boot_attr;
-		};
-
-		if (
-			doc.name === frappe.session.user &&
-			!doc.__unsaved &&
-			frappe.all_timezones &&
-			(hasChanged(doc.language, frappe.boot.user.language) ||
-				hasChanged(doc.time_zone, frappe.boot.time_zone.user) ||
-				hasChanged(doc.desk_theme, frappe.boot.user.desk_theme))
-		) {
-			frappe.msgprint(__("Refreshing..."));
-			window.location.reload();
-		}
-
 		frm.toggle_display(["sb1", "sb3", "modules_access"], false);
 
 		if (!frm.is_new()) {
@@ -335,10 +319,31 @@ frappe.ui.form.on("User", {
 			},
 		});
 	},
-	on_update: function (frm) {
-		if (frappe.boot.time_zone && frappe.boot.time_zone.user !== frm.doc.time_zone) {
-			// Clear cache after saving to refresh the values of boot.
-			frappe.ui.toolbar.clear_cache();
+	after_save: function (frm) {
+		/**
+		 * Checks whether the effective value has changed.
+		 *
+		 * @param {Array.<string>} - Tuple with new override, previous override,
+		 *   and optionally fallback.
+		 * @returns {boolean} - Whether the resulting value has effectively changed
+		 */
+		const has_effectively_changed = ([new_override, prev_override, fallback = undefined]) => {
+			const prev_effective = prev_override || fallback;
+			const new_effective = new_override || fallback;
+			return new_override !== undefined && prev_effective !== new_effective;
+		};
+
+		const doc = frm.doc;
+		const boot = frappe.boot;
+		const attr_tuples = [
+			[doc.language, boot.user.language, boot.sysdefaults.language],
+			[doc.time_zone, boot.time_zone.user, boot.time_zone.system],
+			[doc.desk_theme, boot.user.desk_theme], // No system default.
+		];
+
+		if (doc.name === frappe.session.user && attr_tuples.some(has_effectively_changed)) {
+			frappe.msgprint(__("Refreshing..."));
+			window.location.reload();
 		}
 	},
 });

frappe/core/doctype/user/user.py

@@ -230,7 +230,7 @@ class User(Document):
 			frappe.cache.delete_key("users_for_mentions")
 
 	def has_website_permission(self, ptype, user, verbose=False):
-		"""Returns true if current user is the session user"""
+		"""Return True if current user is the session user."""
 		return self.name == frappe.session.user
 
 	def set_full_name(self):
@@ -686,7 +686,7 @@ class User(Document):
 		)
 
 	def get_blocked_modules(self):
-		"""Returns list of modules blocked for that user"""
+		"""Return list of modules blocked for that user."""
 		return [d.module for d in self.block_modules] if self.block_modules else []
 
 	def validate_user_email_inbox(self):
@@ -1083,7 +1083,7 @@ def user_query(doctype, txt, searchfield, start, page_len, filters):
 
 
 def get_total_users():
-	"""Returns total no. of system users"""
+	"""Return total number of system users."""
 	return flt(
 		frappe.db.sql(
 			"""SELECT SUM(`simultaneous_sessions`)
@@ -1118,7 +1118,7 @@ def get_system_users(exclude_users: Iterable[str] | str | None = None, limit: in
 
 
 def get_active_users():
-	"""Returns No. of system users who logged in, in the last 3 days"""
+	"""Return number of system users who logged in, in the last 3 days."""
 	return frappe.db.sql(
 		"""select count(*) from `tabUser`
 		where enabled = 1 and user_type != 'Website User'
@@ -1131,12 +1131,12 @@ def get_active_users():
 
 
 def get_website_users():
-	"""Returns total no. of website users"""
+	"""Return total number of website users."""
 	return frappe.db.count("User", filters={"enabled": True, "user_type": "Website User"})
 
 
 def get_active_website_users():
-	"""Returns No. of website users who logged in, in the last 3 days"""
+	"""Return number of website users who logged in, in the last 3 days."""
 	return frappe.db.sql(
 		"""select count(*) from `tabUser`
 		where enabled = 1 and user_type = 'Website User'

frappe/core/doctype/user_permission/user_permission.py

@@ -173,7 +173,7 @@ def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len,
 
 
 def get_permitted_documents(doctype):
-	"""Returns permitted documents from the given doctype for the session user"""
+	"""Return permitted documents from the given doctype for the session user."""
 	# sort permissions in a way to make the first permission in the list to be default
 	user_perm_list = sorted(
 		get_user_permissions().get(doctype, []), key=lambda x: x.get("is_default"), reverse=True

frappe/core/doctype/version/version.json

@@ -54,7 +54,7 @@
  "idx": 1,
  "in_create": 1,
  "links": [],
- "modified": "2022-08-03 12:20:53.929691",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Version",
@@ -74,7 +74,7 @@
  ],
  "quick_entry": 1,
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "title_field": "docname",
  "track_changes": 1

frappe/core/notifications.py

@@ -17,7 +17,7 @@ def get_notification_config():
 
 
 def get_things_todo(as_list=False):
-	"""Returns a count of incomplete todos"""
+	"""Return a count of incomplete ToDos."""
 	data = frappe.get_list(
 		"ToDo",
 		fields=["name", "description"] if as_list else "count(*)",
@@ -35,7 +35,7 @@ def get_things_todo(as_list=False):
 
 
 def get_todays_events(as_list: bool = False):
-	"""Returns a count of todays events in calendar"""
+	"""Return a count of today's events in calendar."""
 	from frappe.desk.doctype.event.event import get_events
 	from frappe.utils import nowdate
 

frappe/core/page/permission_manager/permission_manager.py

@@ -109,8 +109,10 @@ def add(parent, role, permlevel):
 
 
 @frappe.whitelist()
-def update(doctype, role, permlevel, ptype, value=None, if_owner=0):
-	"""Update role permission params
+def update(
+	doctype: str, role: str, permlevel: int, ptype: str, value=None, if_owner=0
+) -> str | None:
+	"""Update role permission params.
 
 	Args:
 	        doctype (str): Name of the DocType to update params for
@@ -119,8 +121,8 @@ def update(doctype, role, permlevel, ptype, value=None, if_owner=0):
 	        ptype (str): permission type, example "read", "delete", etc.
 	        value (None, optional): value for ptype, None indicates False
 
-	Returns:
-	        str: Refresh flag is permission is updated successfully
+	Return:
+	        str: Refresh flag if permission is updated successfully
 	"""
 
 	def clear_cache():

frappe/core/utils.py

@@ -7,7 +7,7 @@ import frappe
 
 
 def get_parent_doc(doc):
-	"""Returns document of `reference_doctype`, `reference_doctype`"""
+	"""Return document of `reference_doctype`, `reference_doctype`."""
 	if not hasattr(doc, "parent_doc"):
 		if doc.reference_doctype and doc.reference_name:
 			doc.parent_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name)
@@ -38,8 +38,7 @@ def set_timeline_doc(doc):
 
 
 def find(list_of_dict, match_function):
-	"""Returns a dict in a list of dicts on matching the conditions
-	        provided in match function
+	"""Return a dict in a list of dicts on matching the conditions provided in match function.
 
 	Usage:
 	        list_of_dict = [{'name': 'Suraj'}, {'name': 'Aditya'}]
@@ -54,8 +53,7 @@ def find(list_of_dict, match_function):
 
 
 def find_all(list_of_dict, match_function):
-	"""Returns all matching dicts in a list of dicts.
-	        Uses matching function to filter out the dicts
+	"""Return all matching dicts in a list of dicts. Uses matching function to filter out the dicts.
 
 	Usage:
 	        colored_shapes = [
@@ -86,6 +84,7 @@ def ljust_list(_list, length, fill_word=None):
 	return _list
 
 
-def html2text(html, strip_links=False, wrap=True):
+def html2text(html: str, strip_links=False, wrap=True) -> str:
+	"""Return the given `html` as markdown text."""
 	strip = ["a"] if strip_links else None
 	return md(html, heading_style="ATX", strip=strip, wrap=wrap)

frappe/custom/doctype/client_script/client_script.json

@@ -77,7 +77,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-04-12 12:48:15.717985",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Client Script",
@@ -108,7 +108,7 @@
   }
  ],
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "track_changes": 1
 }

frappe/custom/doctype/custom_field/custom_field.json

@@ -457,7 +457,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2023-10-25 06:55:10.713382",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Custom Field",
@@ -488,7 +488,7 @@
  ],
  "search_fields": "dt,label,fieldtype,options",
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "track_changes": 1
 }

frappe/custom/doctype/custom_field/custom_field.py

@@ -357,7 +357,7 @@ def rename_fieldname(custom_field: str, fieldname: str):
 	if field.is_system_generated:
 		frappe.throw(_("System Generated Fields can not be renamed"))
 	if frappe.db.has_column(parent_doctype, fieldname):
-		frappe.throw(_("Can not rename as fieldname {0} is already present on DocType."))
+		frappe.throw(_("Can not rename as column {0} is already present on DocType.").format(fieldname))
 	if old_fieldname == new_fieldname:
 		frappe.msgprint(_("Old and new fieldnames are same."), alert=True)
 		return

frappe/custom/doctype/customize_form/customize_form.json

@@ -46,6 +46,7 @@
   "column_break_26",
   "email_append_to",
   "sender_field",
+  "sender_name_field",
   "subject_field",
   "section_break_8",
   "sort_field",
@@ -219,7 +220,7 @@
    "depends_on": "email_append_to",
    "fieldname": "sender_field",
    "fieldtype": "Data",
-   "label": "Sender Field",
+   "label": "Sender Email Field",
    "mandatory_depends_on": "email_append_to"
   },
   {
@@ -392,6 +393,12 @@
    "fieldname": "details_tab",
    "fieldtype": "Tab Break",
    "label": "Details"
+  },
+  {
+   "depends_on": "email_append_to",
+   "fieldname": "sender_name_field",
+   "fieldtype": "Data",
+   "label": "Sender Name Field"
   }
  ],
  "hide_toolbar": 1,
@@ -400,7 +407,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2023-11-16 11:23:06.427432",
+ "modified": "2023-12-01 18:18:23.086134",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form",

frappe/custom/doctype/customize_form/customize_form.py

@@ -72,6 +72,7 @@ class CustomizeForm(Document):
 		quick_entry: DF.Check
 		search_fields: DF.Data | None
 		sender_field: DF.Data | None
+		sender_name_field: DF.Data | None
 		show_preview_popup: DF.Check
 		show_title_field_in_link: DF.Check
 		sort_field: DF.Literal
@@ -83,6 +84,7 @@ class CustomizeForm(Document):
 		track_views: DF.Check
 		translated_doctype: DF.Check
 	# end: auto-generated types
+
 	def on_update(self):
 		frappe.db.delete("Singles", {"doctype": "Customize Form"})
 		frappe.db.delete("Customize Form Field")

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -240,8 +240,9 @@ class TestCustomizeForm(FrappeTestCase):
 		# Using Notification Log doctype as it doesn't have any other custom fields
 		d = self.get_customize_form("Notification Log")
 
+		new_document_length = 255
 		document_name = d.get("fields", {"fieldname": "document_name"})[0]
-		document_name.length = 255
+		document_name.length = new_document_length
 		d.run_method("save_customization")
 
 		self.assertEqual(
@@ -250,11 +251,9 @@ class TestCustomizeForm(FrappeTestCase):
 				{"doc_type": "Notification Log", "property": "length", "field_name": "document_name"},
 				"value",
 			),
-			"255",
+			str(new_document_length),
 		)
 
-		self.assertTrue(d.flags.update_db)
-
 		length = frappe.db.sql(
 			"""SELECT character_maximum_length
 			FROM information_schema.columns
@@ -262,7 +261,7 @@ class TestCustomizeForm(FrappeTestCase):
 			AND column_name = 'document_name'"""
 		)[0][0]
 
-		self.assertEqual(length, 255)
+		self.assertEqual(length, new_document_length)
 
 	def test_custom_link(self):
 		try:

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -483,7 +483,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-11-07 13:17:21.373626",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",
@@ -491,6 +491,6 @@
  "owner": "Administrator",
  "permissions": [],
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": []
 }

frappe/database/database.py

@@ -123,7 +123,7 @@ class Database:
 		self._conn.select_db(db_name)
 
 	def get_connection(self):
-		"""Returns a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects"""
+		"""Return a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects."""
 		raise NotImplementedError
 
 	def get_database_size(self):
@@ -160,7 +160,7 @@ class Database:
 		:param ignore_ddl: Catch exception if table, column missing.
 		:param auto_commit: Commit after executing the query.
 		:param update: Update this dict to all rows (if returned `as_dict`).
-		:param run: Returns query without executing it if False.
+		:param run: Return query without executing it if False.
 		:param pluck: Get the plucked field only.
 		:param explain: Print `EXPLAIN` in error log.
 		Examples:
@@ -397,7 +397,7 @@ class Database:
 			raise ImplicitCommitError("This statement can cause implicit commit", query)
 
 	def fetch_as_dict(self) -> list[frappe._dict]:
-		"""Internal. Converts results to dict."""
+		"""Internal. Convert results to dict."""
 		result = self.last_result
 		if result:
 			keys = [column[0] for column in self._cursor.description]
@@ -410,7 +410,7 @@ class Database:
 			frappe.cache.delete_key("db_tables")
 
 	def get_description(self):
-		"""Returns result metadata."""
+		"""Return result metadata."""
 		return self._cursor.description
 
 	@staticmethod
@@ -419,7 +419,7 @@ class Database:
 		return [[value for value in row] for row in res]
 
 	def get(self, doctype, filters=None, as_dict=True, cache=False):
-		"""Returns `get_value` with fieldname='*'"""
+		"""Return `get_value` with fieldname='*'."""
 		return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache)
 
 	def get_value(
@@ -438,7 +438,7 @@ class Database:
 		pluck=False,
 		distinct=False,
 	):
-		"""Returns a document property or list of properties.
+		"""Return a document property or list of properties.
 
 		:param doctype: DocType name.
 		:param filters: Filters like `{"x":"y"}` or name of the document. `None` if Single DocType.
@@ -510,7 +510,7 @@ class Database:
 		distinct=False,
 		limit=None,
 	):
-		"""Returns multiple document properties.
+		"""Return multiple document properties.
 
 		:param doctype: DocType name.
 		:param filters: Filters like `{"x":"y"}` or name of the document.
@@ -926,11 +926,11 @@ class Database:
 		self.set_default(key, val, user)
 
 	def get_global(self, key, user="__global"):
-		"""Returns a global key value."""
+		"""Return a global key value."""
 		return self.get_default(key, user)
 
 	def get_default(self, key, parent="__default"):
-		"""Returns default value as a list if multiple or single"""
+		"""Return default value as a list if multiple or single."""
 		d = self.get_defaults(key, parent)
 		return isinstance(d, list) and d[0] or d
 
@@ -1006,7 +1006,7 @@ class Database:
 		return self.exists("DocField", {"fieldname": fn, "parent": dt})
 
 	def table_exists(self, doctype, cached=True):
-		"""Returns True if table for given doctype exists."""
+		"""Return True if table for given doctype exists."""
 		return f"tab{doctype}" in self.get_tables(cached=cached)
 
 	def has_table(self, doctype):
@@ -1016,7 +1016,7 @@ class Database:
 		raise NotImplementedError
 
 	def a_row_exists(self, doctype):
-		"""Returns True if atleast one row exists."""
+		"""Return True if at least one row exists."""
 		return frappe.get_all(doctype, limit=1, order_by=None, as_list=True)
 
 	def exists(self, dt, dn=None, cache=False):
@@ -1055,7 +1055,7 @@ class Database:
 		return self.get_value(dt, dn, ignore=True, cache=cache, order_by=None)
 
 	def count(self, dt, filters=None, debug=False, cache=False, distinct: bool = True):
-		"""Returns `COUNT(*)` for given DocType and filters."""
+		"""Return `COUNT(*)` for given DocType and filters."""
 		if cache and not filters:
 			cache_count = frappe.cache.get_value(f"doctype:count:{dt}")
 			if cache_count is not None:
@@ -1098,7 +1098,7 @@ class Database:
 		)
 
 	def get_db_table_columns(self, table) -> list[str]:
-		"""Returns list of column names from given table."""
+		"""Return list of column names from given table."""
 		columns = frappe.cache.hget("table_columns", table)
 		if columns is None:
 			information_schema = frappe.qb.Schema("information_schema")
@@ -1116,14 +1116,14 @@ class Database:
 		return columns
 
 	def get_table_columns(self, doctype):
-		"""Returns list of column names from given doctype."""
+		"""Return list of column names from given doctype."""
 		columns = self.get_db_table_columns("tab" + doctype)
 		if not columns:
 			raise self.TableMissingError("DocType", doctype)
 		return columns
 
 	def has_column(self, doctype, column):
-		"""Returns True if column exists in database."""
+		"""Return True if column exists in database."""
 		return column in self.get_table_columns(doctype)
 
 	def has_index(self, table_name, index_name):

frappe/database/db_manager.py

@@ -57,14 +57,15 @@ class DbManager:
 		from frappe.database import get_command
 		from frappe.utils import execute_in_shell
 
-		pv = which("pv")
-
 		command = []
 
-		if pv:
-			command.extend([pv, source, "|"])
-			source = []
-			print("Restoring Database file...")
+		if source.endswith(".gz"):
+			if gzip := which("gzip"):
+				command.extend([gzip, "-cd", source, "|"])
+				source = []
+			else:
+				raise Exception("`gzip` not installed")
+
 		else:
 			source = ["<", source]
 

frappe/database/mariadb/database.py

@@ -191,7 +191,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		}
 
 	def get_database_size(self):
-		"""'Returns database size in MB"""
+		"""Return database size in MB."""
 		db_size = self.sql(
 			"""
 			SELECT `table_schema` as `database_name`,
@@ -281,7 +281,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		)
 
 	def create_global_search_table(self):
-		if not "__global_search" in self.get_tables():
+		if "__global_search" not in self.get_tables():
 			self.sql(
 				"""create table __global_search(
 				doctype varchar(100),
@@ -314,7 +314,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		return "ON DUPLICATE key UPDATE "
 
 	def get_table_columns_description(self, table_name):
-		"""Returns list of column and its description"""
+		"""Return list of columns with descriptions."""
 		return self.sql(
 			"""select
 			column_name as 'name',
@@ -339,7 +339,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		)
 
 	def get_column_type(self, doctype, column):
-		"""Returns column type from database."""
+		"""Return column type from database."""
 		information_schema = frappe.qb.Schema("information_schema")
 		table = get_table_name(doctype)
 
@@ -440,13 +440,13 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 
 			self.commit()
 			db_table.sync()
-			self.begin()
+			self.commit()
 
 	def get_database_list(self):
 		return self.sql("SHOW DATABASES", pluck=True)
 
 	def get_tables(self, cached=True):
-		"""Returns list of tables"""
+		"""Return list of tables."""
 		to_query = not cached
 
 		if cached:

frappe/database/operator_map.py

@@ -17,21 +17,21 @@ def like(key: Field, value: str) -> frappe.qb:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `LIKE`
+	Return:
+	        frappe.qb: `frappe.qb` object with `LIKE`
 	"""
 	return key.like(value)
 
 
 def func_in(key: Field, value: list | tuple) -> frappe.qb:
-	"""Wrapper method for `IN`
+	"""Wrapper method for `IN`.
 
 	Args:
 	        key (str): field
 	        value (Union[int, str]): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `IN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `IN`
 	"""
 	if isinstance(value, str):
 		value = value.split(",")
@@ -39,27 +39,27 @@ def func_in(key: Field, value: list | tuple) -> frappe.qb:
 
 
 def not_like(key: Field, value: str) -> frappe.qb:
-	"""Wrapper method for `NOT LIKE`
+	"""Wrapper method for `NOT LIKE`.
 
 	Args:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `NOT LIKE`
+	Return:
+	        frappe.qb: `frappe.qb` object with `NOT LIKE`
 	"""
 	return key.not_like(value)
 
 
 def func_not_in(key: Field, value: list | tuple | str):
-	"""Wrapper method for `NOT IN`
+	"""Wrapper method for `NOT IN`.
 
 	Args:
 	        key (str): field
 	        value (Union[int, str]): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `NOT IN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `NOT IN`
 	"""
 	if isinstance(value, str):
 		value = value.split(",")
@@ -73,21 +73,21 @@ def func_regex(key: Field, value: str) -> frappe.qb:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `REGEX`
+	Return:
+	        frappe.qb: `frappe.qb` object with `REGEX`
 	"""
 	return key.regex(value)
 
 
 def func_between(key: Field, value: list | tuple) -> frappe.qb:
-	"""Wrapper method for `BETWEEN`
+	"""Wrapper method for `BETWEEN`.
 
 	Args:
 	        key (str): field
 	        value (Union[int, str]): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `BETWEEN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `BETWEEN`
 	"""
 	return key[slice(*value)]
 
@@ -98,14 +98,14 @@ def func_is(key, value):
 
 
 def func_timespan(key: Field, value: str) -> frappe.qb:
-	"""Wrapper method for `TIMESPAN`
+	"""Wrapper method for `TIMESPAN`.
 
 	Args:
 	        key (str): field
 	        value (str): criterion
 
-	Returns:
-	        frappe.qb: `frappe.qb object with `TIMESPAN`
+	Return:
+	        frappe.qb: `frappe.qb` object with `TIMESPAN`
 	"""
 
 	return func_between(key, get_timespan_date_range(value))

frappe/database/postgres/database.py

@@ -197,7 +197,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 		return str(psycopg2.extensions.QuotedString(s))
 
 	def get_database_size(self):
-		"""'Returns database size in MB"""
+		"""Return database size in MB"""
 		db_size = self.sql(
 			"SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", self.db_name, as_dict=True
 		)
@@ -288,7 +288,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 		)
 
 	def create_global_search_table(self):
-		if not "__global_search" in self.get_tables():
+		if "__global_search" not in self.get_tables():
 			self.sql(
 				"""create table "__global_search"(
 				doctype varchar(100),
@@ -329,7 +329,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 
 			self.commit()
 			db_table.sync()
-			self.begin()
+			self.commit()
 
 	@staticmethod
 	def get_on_duplicate_update(key="name"):
@@ -380,7 +380,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 			)
 
 	def get_table_columns_description(self, table_name):
-		"""Returns list of column and its description"""
+		"""Return list of columns with description."""
 		# pylint: disable=W1401
 		return self.sql(
 			"""
@@ -411,7 +411,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 		)
 
 	def get_column_type(self, doctype, column):
-		"""Returns column type from database."""
+		"""Return column type from database."""
 		information_schema = frappe.qb.Schema("information_schema")
 		table = get_table_name(doctype)
 

frappe/database/postgres/setup_db.py

@@ -1,7 +1,7 @@
 import os
 
 import frappe
-from frappe import _
+from frappe.database.db_manager import DbManager
 
 
 def setup_database():
@@ -36,45 +36,16 @@ def bootstrap_database(db_name, verbose, source_sql=None):
 
 
 def import_db_from_sql(source_sql=None, verbose=False):
-	import shlex
-	from shutil import which
-
-	from frappe.database import get_command
-	from frappe.utils import execute_in_shell
-
-	# bootstrap db
+	if verbose:
+		print("Starting database import...")
+	db_name = frappe.conf.db_name
 	if not source_sql:
 		source_sql = os.path.join(os.path.dirname(__file__), "framework_postgres.sql")
-
-	pv = which("pv")
-
-	command = []
-
-	if pv:
-		command.extend([pv, source_sql, "|"])
-		source = []
-		print("Restoring Database file...")
-	else:
-		source = ["-f", source_sql]
-
-	bin, args, bin_name = get_command(
-		host=frappe.conf.db_host,
-		port=frappe.conf.db_port,
-		user=frappe.conf.db_name,
-		password=frappe.conf.db_password,
-		db_name=frappe.conf.db_name,
+	DbManager(frappe.local.db).restore_database(
+		verbose, db_name, source_sql, db_name, frappe.conf.db_password
 	)
-
-	if not bin:
-		frappe.throw(
-			_("{} not found in PATH! This is required to restore the database.").format(bin_name),
-			exc=frappe.ExecutableNotFound,
-		)
-	command.append(bin)
-	command.append(shlex.join(args))
-	command.extend(source)
-	execute_in_shell(" ".join(command), check_exit_code=True, verbose=verbose)
-	frappe.cache.delete_keys("")  # Delete all keys associated with this site.
+	if verbose:
+		print("Imported from database %s" % source_sql)
 
 
 def get_root_connection(root_login=None, root_password=None):

frappe/database/query.py

@@ -218,7 +218,7 @@ class Engine:
 			self.query = self.query.where(operator_fn(_field, _value))
 
 	def get_function_object(self, field: str) -> "Function":
-		"""Expects field to look like 'SUM(*)' or 'name' or something similar. Returns PyPika Function object"""
+		"""Return PyPika Function object. Expect field to look like 'SUM(*)' or 'name' or something similar."""
 		func = field.split("(", maxsplit=1)[0].capitalize()
 		args_start, args_end = len(func) + 1, field.index(")")
 		args = field[args_start:args_end].split(",")

frappe/defaults.py

@@ -79,7 +79,7 @@ def is_a_user_permission_key(key):
 
 
 def not_in_user_permission(key, value, user=None):
-	# returns true or false based on if value exist in user permission
+	# return true or false based on if value exist in user permission
 	user = user or frappe.session.user
 	user_permission = get_user_permissions(user).get(frappe.unscrub(key)) or []
 

frappe/desk/calendar.py

@@ -19,7 +19,7 @@ def update_event(args, field_map):
 
 
 def get_event_conditions(doctype, filters=None):
-	"""Returns SQL conditions with user permissions and filters for event queries"""
+	"""Return SQL conditions with user permissions and filters for event queries."""
 	from frappe.desk.reportview import get_filters_cond
 
 	if not frappe.has_permission(doctype):

frappe/desk/desktop.py

@@ -68,7 +68,7 @@ class Workspace:
 		)
 
 	def is_permitted(self):
-		"""Returns true if Has Role is not set or the user is allowed."""
+		"""Return true if `Has Role` is not set or the user is allowed."""
 		from frappe.utils import has_common
 
 		allowed = [d.role for d in self.doc.roles]
@@ -383,13 +383,12 @@ class Workspace:
 @frappe.whitelist()
 @frappe.read_only()
 def get_desktop_page(page):
-	"""Applies permissions, customizations and returns the configruration for a page
-	on desk.
+	"""Apply permissions, customizations and return the configuration for a page on desk.
 
 	Args:
 	        page (json): page data
 
-	Returns:
+	Return:
 	        dict: dictionary of cards, charts and shortcuts to be displayed on website
 	"""
 	try:
@@ -503,7 +502,7 @@ def get_custom_doctype_list(module):
 
 
 def get_custom_report_list(module):
-	"""Returns list on new style reports for modules."""
+	"""Return list on new style reports for modules."""
 	reports = frappe.get_all(
 		"Report",
 		fields=["name", "ref_doctype", "report_type"],
@@ -617,14 +616,14 @@ def new_widget(config, doctype, parentfield):
 
 
 def prepare_widget(config, doctype, parentfield):
-	"""Create widget child table entries with parent details
+	"""Create widget child table entries with parent details.
 
 	Args:
 	        config (dict): Dictionary containing widget config
 	        doctype (string): Doctype name of the child table
 	        parentfield (string): Parent field for the child table
 
-	Returns:
+	Return:
 	        TYPE: List of Document objects
 	"""
 	if not config:

frappe/desk/doctype/bulk_update/bulk_update.py

@@ -6,6 +6,7 @@ from frappe import _
 from frappe.core.doctype.submission_queue.submission_queue import queue_submission
 from frappe.model.document import Document
 from frappe.utils import cint
+from frappe.utils.deprecations import deprecated
 from frappe.utils.scheduler import is_scheduler_inactive
 
 
@@ -24,6 +25,7 @@ class BulkUpdate(Document):
 		limit: DF.Int
 		update_value: DF.SmallText
 	# end: auto-generated types
+
 	@frappe.whitelist()
 	def bulk_update(self):
 		self.check_permission("write")
@@ -45,12 +47,12 @@ class BulkUpdate(Document):
 
 
 @frappe.whitelist()
-def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):
+def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None, task_id=None):
 	if isinstance(docnames, str):
 		docnames = frappe.parse_json(docnames)
 
 	if len(docnames) < 20:
-		return _bulk_action(doctype, docnames, action, data)
+		return _bulk_action(doctype, docnames, action, data, task_id)
 	elif len(docnames) <= 500:
 		frappe.msgprint(_("Bulk operation is enqueued in background."), alert=True)
 		frappe.enqueue(
@@ -59,6 +61,7 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):
 			docnames=docnames,
 			action=action,
 			data=data,
+			task_id=task_id,
 			queue="short",
 			timeout=1000,
 		)
@@ -68,14 +71,15 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):
 		)
 
 
-def _bulk_action(doctype, docnames, action, data):
+def _bulk_action(doctype, docnames, action, data, task_id=None):
 	if data:
 		data = frappe.parse_json(data)
 
 	failed = []
+	num_documents = len(docnames)
 
-	for i, d in enumerate(docnames, 1):
-		doc = frappe.get_doc(doctype, d)
+	for idx, docname in enumerate(docnames, 1):
+		doc = frappe.get_doc(doctype, docname)
 		try:
 			message = ""
 			if action == "submit" and doc.docstatus.is_draft():
@@ -93,17 +97,23 @@ def _bulk_action(doctype, docnames, action, data):
 				doc.save()
 				message = _("Updating {0}").format(doctype)
 			else:
-				failed.append(d)
+				failed.append(docname)
 			frappe.db.commit()
-			show_progress(docnames, message, i, d)
+			frappe.publish_progress(
+				percent=idx / num_documents * 100,
+				title=message,
+				description=docname,
+				task_id=task_id,
+			)
 
 		except Exception:
-			failed.append(d)
+			failed.append(docname)
 			frappe.db.rollback()
 
 	return failed
 
 
+@deprecated
 def show_progress(docnames, message, i, description):
 	n = len(docnames)
 	frappe.publish_progress(float(i) * 100 / n, title=message, description=description)

frappe/desk/doctype/dashboard_chart/dashboard_chart.py

@@ -317,7 +317,7 @@ def get_result(data, timegrain, from_date, to_date, chart_type):
 				d[1] += data[data_index][1]
 				count += data[data_index][2]
 				data_index += 1
-			if chart_type == "Average" and not count == 0:
+			if chart_type == "Average" and count != 0:
 				d[1] = d[1] / count
 			if chart_type == "Count":
 				d[1] = count

frappe/desk/doctype/dashboard_settings/dashboard_settings.py

@@ -51,7 +51,7 @@ def save_chart_config(reset, config, chart_name):
 		chart_config[chart_name] = {}
 	else:
 		config = frappe.parse_json(config)
-		if not chart_name in chart_config:
+		if chart_name not in chart_config:
 			chart_config[chart_name] = {}
 		chart_config[chart_name].update(config)
 

frappe/desk/doctype/list_view_settings/list_view_settings.py

@@ -96,7 +96,7 @@ def get_default_listview_fields(doctype):
 	fields = [f.get("fieldname") for f in doctype_json.get("fields") if f.get("in_list_view")]
 
 	if meta.title_field:
-		if not meta.title_field.strip() in fields:
+		if meta.title_field.strip() not in fields:
 			fields.append(meta.title_field.strip())
 
 	return fields

frappe/desk/doctype/note/note.json

@@ -86,7 +86,7 @@
  "icon": "fa fa-file-text",
  "idx": 1,
  "links": [],
- "modified": "2023-08-28 20:23:59.424943",
+ "modified": "2023-12-08 15:52:37.525003",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Note",
@@ -141,7 +141,7 @@
  ],
  "quick_entry": 1,
  "sort_field": "modified",
- "sort_order": "ASC",
+ "sort_order": "DESC",
  "states": [],
  "title_field": "title",
  "track_changes": 1

frappe/desk/doctype/notification_settings/notification_settings.js

@@ -3,11 +3,6 @@
 
 frappe.ui.form.on("Notification Settings", {
 	onload: (frm) => {
-		frappe.breadcrumbs.add({
-			label: __("Settings"),
-			route: "#modules/Settings",
-			type: "Custom",
-		});
 		frm.set_query("subscribed_documents", () => {
 			return {
 				filters: {

frappe/desk/doctype/notification_settings/notification_settings.json

@@ -12,6 +12,7 @@
   "enable_email_notifications",
   "enable_email_mention",
   "enable_email_assignment",
+  "enable_email_threads_on_assigned_document",
   "enable_email_energy_point",
   "enable_email_share",
   "enable_email_event_reminders",
@@ -105,12 +106,20 @@
    "fieldname": "enable_email_event_reminders",
    "fieldtype": "Check",
    "label": "Event Reminders"
+  },
+  {
+   "default": "1",
+   "depends_on": "enable_email_notifications",
+   "description": "Get notified when an email is received on any of the documents assigned to you.",
+   "fieldname": "enable_email_threads_on_assigned_document",
+   "fieldtype": "Check",
+   "label": "Email Threads on Assigned Document"
   }
  ],
  "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-11-24 14:45:31.931154",
+ "modified": "2023-12-01 12:46:15.490640",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Notification Settings",
@@ -132,5 +141,6 @@
  "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/desk/doctype/notification_settings/notification_settings.py

@@ -23,6 +23,7 @@ class NotificationSettings(Document):
 		enable_email_mention: DF.Check
 		enable_email_notifications: DF.Check
 		enable_email_share: DF.Check
+		enable_email_threads_on_assigned_document: DF.Check
 		enabled: DF.Check
 		energy_points_system_notifications: DF.Check
 		seen: DF.Check

frappe/desk/doctype/route_history/route_history.json

@@ -1,5 +1,6 @@
 {
  "actions": [],
+ "allow_copy": 1,
  "creation": "2018-10-05 11:26:04.601113",
  "doctype": "DocType",
  "editable_grid": 1,
@@ -13,7 +14,9 @@
    "fieldname": "route",
    "fieldtype": "Data",
    "in_list_view": 1,
-   "label": "Route"
+   "label": "Route",
+   "no_copy": 1,
+   "read_only": 1
   },
   {
    "fieldname": "user",
@@ -21,30 +24,29 @@
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "User",
-   "options": "User"
+   "no_copy": 1,
+   "options": "User",
+   "read_only": 1
   }
  ],
+ "in_create": 1,
  "links": [],
- "modified": "2022-06-13 05:48:56.967244",
+ "modified": "2023-12-04 04:41:32.448331",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Route History",
  "owner": "Administrator",
  "permissions": [
   {
-   "create": 1,
-   "delete": 1,
    "email": 1,
    "export": 1,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "share": 1,
-   "write": 1
+   "share": 1
   }
  ],
- "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
  "states": [],

frappe/desk/doctype/route_history/route_history.py

@@ -18,6 +18,7 @@ class RouteHistory(Document):
 		route: DF.Data | None
 		user: DF.Link | None
 	# end: auto-generated types
+
 	@staticmethod
 	def clear_old_logs(days=30):
 		from frappe.query_builder import Interval

frappe/desk/doctype/system_console/system_console.py

@@ -28,7 +28,7 @@ class SystemConsole(Document):
 		try:
 			frappe.local.debug_log = []
 			if self.type == "Python":
-				safe_exec(self.console)
+				safe_exec(self.console, script_filename="System Console")
 				self.output = "\n".join(frappe.debug_log)
 			elif self.type == "SQL":
 				self.output = frappe.as_json(read_sql(self.console, as_dict=1))

frappe/desk/doctype/tag/tag.py

@@ -77,33 +77,33 @@ class DocTags:
 		self.dt = dt
 
 	def get_tag_fields(self):
-		"""returns tag_fields property"""
+		"""Return `tag_fields` property."""
 		return frappe.db.get_value("DocType", self.dt, "tag_fields")
 
 	def get_tags(self, dn):
-		"""returns tag for a particular item"""
+		"""Return tag for a particular item."""
 		return (frappe.db.get_value(self.dt, dn, "_user_tags", ignore=1) or "").strip()
 
 	def add(self, dn, tag):
-		"""add a new user tag"""
+		"""Add a new user tag."""
 		tl = self.get_tags(dn).split(",")
-		if not tag in tl:
+		if tag not in tl:
 			tl.append(tag)
 			if not frappe.db.exists("Tag", tag):
 				frappe.get_doc({"doctype": "Tag", "name": tag}).insert(ignore_permissions=True)
 			self.update(dn, tl)
 
 	def remove(self, dn, tag):
-		"""remove a user tag"""
+		"""Remove a user tag."""
 		tl = self.get_tags(dn).split(",")
 		self.update(dn, filter(lambda x: x.lower() != tag.lower(), tl))
 
 	def remove_all(self, dn):
-		"""remove all user tags (call before delete)"""
+		"""Remove all user tags (call before delete)."""
 		self.update(dn, [])
 
 	def update(self, dn, tl):
-		"""updates the _user_tag column in the table"""
+		"""Update the `_user_tag` column in the table."""
 
 		if not tl:
 			tags = ""
@@ -128,16 +128,15 @@ class DocTags:
 				raise
 
 	def setup(self):
-		"""adds the _user_tags column if not exists"""
+		"""Add the `_user_tags` column if not exists."""
 		from frappe.database.schema import add_column
 
 		add_column(self.dt, "_user_tags", "Data")
 
 
 def delete_tags_for_document(doc):
-	"""
-	Delete the Tag Link entry of a document that has
-	been deleted
+	"""Delete the Tag Link entry of a document that has been deleted.
+
 	:param doc: Deleted document
 	"""
 	if not frappe.db.table_exists("Tag Link"):
@@ -147,7 +146,7 @@ def delete_tags_for_document(doc):
 
 
 def update_tags(doc, tags):
-	"""Adds tags for documents
+	"""Add tags for documents.
 
 	:param doc: Document to be added to global tags
 	"""
@@ -181,8 +180,8 @@ def update_tags(doc, tags):
 
 @frappe.whitelist()
 def get_documents_for_tag(tag):
-	"""
-	Search for given text in Tag Link
+	"""Search for given text in Tag Link.
+
 	:param tag: tag to be searched
 	"""
 	# remove hastag `#` from tag

frappe/desk/doctype/todo/todo.py

@@ -136,7 +136,7 @@ class ToDo(Document):
 
 	@classmethod
 	def get_owners(cls, filters=None):
-		"""Returns list of owners after applying filters on todo's."""
+		"""Return list of owners after applying filters on ToDos."""
 		rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=["allocated_to"])
 		return [parse_addr(row.allocated_to)[1] for row in rows if row.allocated_to]