From fc0ba518945102b836794fc539ea6bd42af60cc1 Mon Sep 17 00:00:00 2001
From: marination <maricadsouza221197@gmail.com>
Date: Thu, 21 Dec 2023 16:15:56 +0100
Subject: [PATCH 01/64] fix: Disappearing letterhead header in pdf

- fix: Make sure 'header-html' is extracted from document after it is rendered as html and loaded in wkhtmltopdf options, else it goes missing
---
 frappe/utils/pdf.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py
index 24b6896620..b7a8557b48 100644
--- a/frappe/utils/pdf.py
+++ b/frappe/utils/pdf.py
@@ -227,7 +227,7 @@ def read_options_from_html(html):
 	return str(soup), options
 
 
-def prepare_header_footer(soup):
+def prepare_header_footer(soup: BeautifulSoup):
 	options = {}
 
 	head = soup.find("head").contents
@@ -240,10 +240,6 @@ def prepare_header_footer(soup):
 	for html_id in ("header-html", "footer-html"):
 		content = soup.find(id=html_id)
 		if content:
-			# there could be multiple instances of header-html/footer-html
-			for tag in soup.find_all(id=html_id):
-				tag.extract()
-
 			toggle_visible_pdf(content)
 			id_map = {"header-html": "pdf_header_html", "footer-html": "pdf_footer_html"}
 			hook_func = frappe.get_hooks(id_map.get(html_id))
@@ -256,6 +252,10 @@ def prepare_header_footer(soup):
 				css=css,
 			)
 
+			# there could be multiple instances of header-html/footer-html
+			for tag in soup.find_all(id=html_id):
+				tag.extract()
+
 			# create temp file
 			fname = os.path.join("/tmp", f"frappe-pdf-{frappe.generate_hash()}.html")
 			with open(fname, "wb") as f:

From 29e761671dc73edcc3b0a9745d825b8851a34e34 Mon Sep 17 00:00:00 2001
From: marination <maricadsouza221197@gmail.com>
Date: Thu, 21 Dec 2023 17:25:19 +0100
Subject: [PATCH 02/64] feat: Letterhead scripts

- feat: Allow script injection into header/footer.html to allow manipulation of styles using page numbers/args received by wkhtmltopdf
- misc: also validate letterhead scripts
- Include scripts in printview as well as pdf
- Add helper instructions & re-arrange fields
---
 frappe/__init__.py                            |  6 ++-
 .../doctype/letter_head/letter_head.js        | 51 +++++++++++++++++++
 .../doctype/letter_head/letter_head.json      | 35 ++++++++++++-
 .../print_formats/pdf_header_footer.html      | 16 +++++-
 frappe/utils/pdf.py                           | 26 ++++++----
 frappe/www/printview.html                     | 16 +++---
 frappe/www/printview.py                       | 33 ++++++++++--
 7 files changed, 157 insertions(+), 26 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 3d31adbcfa..7c41c21b51 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -2116,7 +2116,9 @@ def get_print(
 		pdf_options["password"] = password
 
 	html = get_response_content("printview")
-	return get_pdf(html, options=pdf_options, output=output) if as_pdf else html
+	return (
+		get_pdf(html, options=pdf_options, output=output, letterhead=letterhead) if as_pdf else html
+	)
 
 
 def attach_print(
@@ -2155,7 +2157,7 @@ def attach_print(
 			ext = ".pdf"
 			kwargs["as_pdf"] = True
 			content = (
-				get_pdf(html, options={"password": password} if password else None)
+				get_pdf(html, options={"password": password} if password else None, letterhead=letterhead)
 				if html
 				else get_print(doctype, name, **kwargs)
 			)
diff --git a/frappe/printing/doctype/letter_head/letter_head.js b/frappe/printing/doctype/letter_head/letter_head.js
index 55d97cf37f..a1ef08bce9 100644
--- a/frappe/printing/doctype/letter_head/letter_head.js
+++ b/frappe/printing/doctype/letter_head/letter_head.js
@@ -2,7 +2,58 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Letter Head", {
+	setup(frm) {
+		frm.get_field("instructions").html(INSTRUCTIONS);
+	},
+
 	refresh: function (frm) {
 		frm.flag_public_attachments = true;
 	},
+
+	validate: (frm) => {
+		["header_script", "footer_script"].forEach((field) => {
+			if (!frm.doc[field]) return;
+
+			try {
+				eval(frm.doc[field]);
+			} catch (e) {
+				frappe.throw({
+					title: __("Error in Header/Footer Script"),
+					indicator: "orange",
+					message: '<pre class="small"><code>' + e.stack + "</code></pre>",
+				});
+			}
+		});
+	},
 });
+
+const INSTRUCTIONS = `<h4>Letter Head Scripts</h4>
+<p>Header/Footer scripts can be used to add dynamic behaviours.</p>
+<pre>
+<code>
+// The following Header Script will add the current date to an element in 'Header HTML' with class "header-content"
+var el = document.getElementsByClassName("header-content");
+if (el.length > 0) {
+	el[0].textContent += " " + new Date().toGMTString();
+}
+</code>
+</pre>
+<p>You can also access wkhtmltopdf variables (valid only in PDF print):</p>
+<pre>
+<code>
+// Get Header and Footer wkhtmltopdf variables
+// Snippet and more variables: https://wkhtmltopdf.org/usage/wkhtmltopdf.txt
+var vars = {};
+var query_strings_from_url = document.location.search.substring(1).split('&');
+for (var query_string in query_strings_from_url) {
+	if (query_strings_from_url.hasOwnProperty(query_string)) {
+		var temp_var = query_strings_from_url[query_string].split('=', 2);
+		vars[temp_var[0]] = decodeURI(temp_var[1]);
+	}
+}
+var el = document.getElementsByClassName("header-content");
+if (el.length > 0 && vars["page"] == 1) {
+	el[0].textContent += " : " + vars["date"];
+}
+</code>
+</pre>`;
diff --git a/frappe/printing/doctype/letter_head/letter_head.json b/frappe/printing/doctype/letter_head/letter_head.json
index 021f79ca93..4ffca134f2 100644
--- a/frappe/printing/doctype/letter_head/letter_head.json
+++ b/frappe/printing/doctype/letter_head/letter_head.json
@@ -26,7 +26,11 @@
   "footer_image",
   "footer_image_height",
   "footer_image_width",
-  "footer_align"
+  "footer_align",
+  "scripts_section",
+  "header_script",
+  "footer_script",
+  "instructions"
  ],
  "fields": [
   {
@@ -162,13 +166,40 @@
    "fieldtype": "Select",
    "label": "Footer Based On",
    "options": "Image\nHTML"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && doc.source==='HTML'",
+   "fieldname": "header_script",
+   "fieldtype": "Code",
+   "label": "Header Script",
+   "options": "Javascript"
+  },
+  {
+   "depends_on": "eval:!doc.__islocal && doc.footer_source==='HTML'",
+   "fieldname": "footer_script",
+   "fieldtype": "Code",
+   "label": "Footer Script",
+   "options": "Javascript"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "eval: doc.header_script || doc.footer_script",
+   "fieldname": "scripts_section",
+   "fieldtype": "Section Break",
+   "label": "Scripts"
+  },
+  {
+   "fieldname": "instructions",
+   "fieldtype": "HTML",
+   "label": "Instructions",
+   "read_only": 1
   }
  ],
  "icon": "fa fa-font",
  "idx": 1,
  "links": [],
  "max_attachments": 3,
- "modified": "2023-12-08 15:52:37.525003",
+ "modified": "2023-12-21 16:19:37.525003",
  "modified_by": "Administrator",
  "module": "Printing",
  "name": "Letter Head",
diff --git a/frappe/templates/print_formats/pdf_header_footer.html b/frappe/templates/print_formats/pdf_header_footer.html
index 189cf43cd9..91973efb70 100644
--- a/frappe/templates/print_formats/pdf_header_footer.html
+++ b/frappe/templates/print_formats/pdf_header_footer.html
@@ -56,14 +56,28 @@
 					}
 				}
 			}
+
+			function custom_load() {
+				if (window.custom_header_footer_script) {
+					window.custom_header_footer_script();
+				}
+			}
 		</script>
 
+		{% if custom_header_footer_script -%}
+		<script>
+			window.custom_header_footer_script = function() {
+				{{ custom_header_footer_script }}
+			}
+		</script>
+		{%- endif %}
+
 		{% for tag in styles -%}
 			{{ tag | string }}
 		{%- endfor %}
 
 	</head>
-	<body onload="subst()">
+	<body onload="subst(); custom_load();">
 		<div class="print-format">
 			<div class="wrapper">
 				{% for tag in content -%}
diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py
index b7a8557b48..1f766e4e66 100644
--- a/frappe/utils/pdf.py
+++ b/frappe/utils/pdf.py
@@ -24,7 +24,7 @@ PDF_CONTENT_ERRORS = [
 ]
 
 
-def pdf_header_html(soup, head, content, styles, html_id, css):
+def pdf_header_html(soup, head, content, styles, html_id, css, script=None):
 	return frappe.render_template(
 		"templates/print_formats/pdf_header_footer.html",
 		{
@@ -35,6 +35,7 @@ def pdf_header_html(soup, head, content, styles, html_id, css):
 			"css": css,
 			"lang": frappe.local.lang,
 			"layout_direction": "rtl" if is_rtl() else "ltr",
+			"custom_header_footer_script": script,
 		},
 	)
 
@@ -66,15 +67,15 @@ def _guess_template_error_line_number(template) -> int | None:
 				return frame.lineno
 
 
-def pdf_footer_html(soup, head, content, styles, html_id, css):
+def pdf_footer_html(soup, head, content, styles, html_id, css, script=None):
 	return pdf_header_html(
-		soup=soup, head=head, content=content, styles=styles, html_id=html_id, css=css
+		soup=soup, head=head, content=content, styles=styles, html_id=html_id, css=css, script=script
 	)
 
 
-def get_pdf(html, options=None, output: PdfWriter | None = None):
+def get_pdf(html, options=None, output: PdfWriter | None = None, letterhead: str | None = None):
 	html = scrub_urls(html)
-	html, options = prepare_options(html, options)
+	html, options = prepare_options(html, options, letterhead=letterhead)
 
 	options.update({"disable-javascript": "", "disable-local-file-access": ""})
 
@@ -133,7 +134,7 @@ def get_file_data_from_writer(writer_obj):
 	return stream.read()
 
 
-def prepare_options(html, options):
+def prepare_options(html, options, letterhead: str | None = None):
 	if not options:
 		options = {}
 
@@ -155,7 +156,7 @@ def prepare_options(html, options):
 	if not options.get("margin-left"):
 		options["margin-left"] = "15mm"
 
-	html, html_options = read_options_from_html(html)
+	html, html_options = read_options_from_html(html, letterhead=letterhead)
 	options.update(html_options or {})
 
 	# cookies
@@ -196,11 +197,11 @@ def get_cookie_options():
 	return options
 
 
-def read_options_from_html(html):
+def read_options_from_html(html, letterhead: str | None = None):
 	options = {}
 	soup = BeautifulSoup(html, "html5lib")
 
-	options.update(prepare_header_footer(soup))
+	options.update(prepare_header_footer(soup, letterhead=letterhead))
 
 	toggle_visible_pdf(soup)
 
@@ -227,7 +228,7 @@ def read_options_from_html(html):
 	return str(soup), options
 
 
-def prepare_header_footer(soup: BeautifulSoup):
+def prepare_header_footer(soup: BeautifulSoup, letterhead: str | None = None):
 	options = {}
 
 	head = soup.find("head").contents
@@ -240,7 +241,11 @@ def prepare_header_footer(soup: BeautifulSoup):
 	for html_id in ("header-html", "footer-html"):
 		content = soup.find(id=html_id)
 		if content:
+			field = "header_script" if html_id == "header-html" else "footer_script"
+			script = frappe.db.get_value("Letter Head", letterhead, field) if letterhead else None
+
 			toggle_visible_pdf(content)
+
 			id_map = {"header-html": "pdf_header_html", "footer-html": "pdf_footer_html"}
 			hook_func = frappe.get_hooks(id_map.get(html_id))
 			html = frappe.get_attr(hook_func[-1])(
@@ -250,6 +255,7 @@ def prepare_header_footer(soup: BeautifulSoup):
 				styles=styles,
 				html_id=html_id,
 				css=css,
+				script=script,
 			)
 
 			# there could be multiple instances of header-html/footer-html
diff --git a/frappe/www/printview.html b/frappe/www/printview.html
index a377fab8ea..9841bd045a 100644
--- a/frappe/www/printview.html
+++ b/frappe/www/printview.html
@@ -31,14 +31,18 @@
 		document.addEventListener('DOMContentLoaded', () => {
 			const page_div = document.querySelector('.page-break');
 
-			page_div.style.display = 'flex';
-			page_div.style.flexDirection = 'column';
+			if (page_div) {
+				page_div.style.display = 'flex';
+				page_div.style.flexDirection = 'column';
+			}
 
 			const footer_html = document.getElementById('footer-html');
-			footer_html.classList.add('hidden-pdf');
-			footer_html.classList.remove('visible-pdf');
-			footer_html.style.order = 1;
-			footer_html.style.marginTop = '20px';
+			if (footer_html)  {
+				footer_html.classList.add('hidden-pdf');
+				footer_html.classList.remove('visible-pdf');
+				footer_html.style.order = 1;
+				footer_html.style.marginTop = '20px';
+			}
 		});
 	</script>
 </body>
diff --git a/frappe/www/printview.py b/frappe/www/printview.py
index 2095e7aa76..01367df2e6 100644
--- a/frappe/www/printview.py
+++ b/frappe/www/printview.py
@@ -198,11 +198,23 @@ def get_rendered_template(
 		letter_head.content = frappe.utils.jinja.render_template(
 			letter_head.content, {"doc": doc.as_dict()}
 		)
+		if letter_head.header_script:
+			letter_head.content += f"""
+				<script>
+					{ letter_head.header_script }
+				</script>
+			"""
 
 	if letter_head.footer:
 		letter_head.footer = frappe.utils.jinja.render_template(
 			letter_head.footer, {"doc": doc.as_dict()}
 		)
+		if letter_head.footer_script:
+			letter_head.footer += f"""
+				<script>
+					{ letter_head.footer_script }
+				</script>
+			"""
 
 	convert_markdown(doc)
 
@@ -391,13 +403,24 @@ def validate_key(key: str, doc: "Document") -> None:
 def get_letter_head(doc: "Document", no_letterhead: bool, letterhead: str | None = None) -> dict:
 	if no_letterhead:
 		return {}
-	if letterhead:
-		return frappe.db.get_value("Letter Head", letterhead, ["content", "footer"], as_dict=True)
-	if doc.get("letter_head"):
-		return frappe.db.get_value("Letter Head", doc.letter_head, ["content", "footer"], as_dict=True)
+
+	letterhead_name = letterhead or doc.get("letter_head")
+	if letterhead_name:
+		return frappe.db.get_value(
+			"Letter Head",
+			letterhead_name,
+			["content", "footer", "header_script", "footer_script"],
+			as_dict=True,
+		)
 	else:
 		return (
-			frappe.db.get_value("Letter Head", {"is_default": 1}, ["content", "footer"], as_dict=True) or {}
+			frappe.db.get_value(
+				"Letter Head",
+				{"is_default": 1},
+				["content", "footer", "header_script", "footer_script"],
+				as_dict=True,
+			)
+			or {}
 		)
 
 

From 94378b6566e4dd738ebcb9b7f3718c130ccb0e61 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Wed, 17 Jan 2024 16:55:48 +0100
Subject: [PATCH 03/64] chore: simplify frappe.connect and use site config for
 bootstrapping

---
 frappe/__init__.py                   | 9 ++++++++-
 frappe/database/__init__.py          | 6 +++---
 frappe/database/mariadb/setup_db.py  | 6 +++---
 frappe/database/postgres/setup_db.py | 9 +++++----
 frappe/installer.py                  | 1 -
 5 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 289e5c6e3d..e930b8d500 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -280,7 +280,7 @@ def connect(
 	"""Connect to site database instance.
 
 	:param site: (Deprecated) If site is given, calls `frappe.init`.
-	:param db_name: Optional. Will use from `site_config.json`.
+	:param db_name: (Deprecated) Optional. Will use from `site_config.json`.
 	:param set_admin_as_user: Set Administrator as current user.
 	"""
 	from frappe.database import get_db
@@ -293,6 +293,13 @@ def connect(
 			"Instead, explicitly invoke frappe.init(site) prior to calling frappe.connect(), if initializing the site is necessary."
 		)
 		init(site)
+	if db_name:
+		from frappe.utils.deprecations import deprecation_warning
+
+		deprecation_warning(
+			"Calling frappe.connect with the db_name argument is deprecated and will be removed in next major version. "
+			"Instead, explicitly invoke frappe.init(site) with the right config prior to calling frappe.connect(), if necessary."
+		)
 
 	local.db = get_db(
 		host=local.conf.db_host,
diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py
index 530bd4c700..93241d6793 100644
--- a/frappe/database/__init__.py
+++ b/frappe/database/__init__.py
@@ -23,17 +23,17 @@ def setup_database(force, verbose=None, no_mariadb_socket=False):
 		)
 
 
-def bootstrap_database(db_name, verbose=None, source_sql=None):
+def bootstrap_database(verbose=None, source_sql=None):
 	import frappe
 
 	if frappe.conf.db_type == "postgres":
 		import frappe.database.postgres.setup_db
 
-		return frappe.database.postgres.setup_db.bootstrap_database(db_name, verbose, source_sql)
+		return frappe.database.postgres.setup_db.bootstrap_database(verbose, source_sql)
 	else:
 		import frappe.database.mariadb.setup_db
 
-		return frappe.database.mariadb.setup_db.bootstrap_database(db_name, verbose, source_sql)
+		return frappe.database.mariadb.setup_db.bootstrap_database(verbose, source_sql)
 
 
 def drop_user_and_database(db_name, db_user):
diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py
index 7cf567c504..348c16529e 100644
--- a/frappe/database/mariadb/setup_db.py
+++ b/frappe/database/mariadb/setup_db.py
@@ -73,17 +73,17 @@ def drop_user_and_database(
 	dbman.delete_user(db_user)
 
 
-def bootstrap_database(db_name, verbose, source_sql=None):
+def bootstrap_database(verbose, source_sql=None):
 	import sys
 
-	frappe.connect(db_name=db_name)
+	frappe.connect()
 	if not check_database_settings():
 		print("Database settings do not match expected values; stopping database setup.")
 		sys.exit(1)
 
 	import_db_from_sql(source_sql, verbose)
-	frappe.connect(db_name=db_name)
 
+	frappe.connect()
 	if "tabDefaultValue" not in frappe.db.get_tables(cached=False):
 		from click import secho
 
diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py
index 855f31b89c..ba45af4cad 100644
--- a/frappe/database/postgres/setup_db.py
+++ b/frappe/database/postgres/setup_db.py
@@ -29,11 +29,12 @@ def setup_database():
 	root_conn.close()
 
 
-def bootstrap_database(db_name, verbose, source_sql=None):
-	frappe.connect(db_name=db_name)
-	import_db_from_sql(source_sql, verbose)
-	frappe.connect(db_name=db_name)
+def bootstrap_database(verbose, source_sql=None):
 
+	frappe.connect()
+	import_db_from_sql(source_sql, verbose)
+
+	frappe.connect()
 	if "tabDefaultValue" not in frappe.db.get_tables():
 		import sys
 
diff --git a/frappe/installer.py b/frappe/installer.py
index 2f7138cb48..fce104ad99 100644
--- a/frappe/installer.py
+++ b/frappe/installer.py
@@ -170,7 +170,6 @@ def install_db(
 		setup_database(force, verbose, no_mariadb_socket)
 
 	bootstrap_database(
-		db_name=frappe.conf.db_name,
 		verbose=verbose,
 		source_sql=source_sql,
 	)

From cc6ff2994f2bcd5b63bcae9c22cd7882e36f0dda Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Fri, 19 Jan 2024 16:13:09 +0100
Subject: [PATCH 04/64] fix: make render_include work without `local` object

frappe.throw relies on local, making it impossible to use this function without an active site.
---
 frappe/model/utils/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/model/utils/__init__.py b/frappe/model/utils/__init__.py
index 153a42ec12..d0015101b1 100644
--- a/frappe/model/utils/__init__.py
+++ b/frappe/model/utils/__init__.py
@@ -66,7 +66,7 @@ def render_include(content):
 		if "{% include" in content:
 			paths = INCLUDE_DIRECTIVE_PATTERN.findall(content)
 			if not paths:
-				frappe.throw(_("Invalid include path"), InvalidIncludePath)
+				raise InvalidIncludePath
 
 			for path in paths:
 				app, app_path = path.split("/", 1)

From 368283769f06f8ca2f3110f4fbf32facbb1e5535 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Fri, 19 Jan 2024 16:16:38 +0100
Subject: [PATCH 05/64] refactor: move `extract_messages_from_code` to a
 separate utils file

In order to avoid circular imports.
---
 frappe/gettext/extractors/utils.py | 80 ++++++++++++++++++++++++++++++
 frappe/translate.py                | 79 +----------------------------
 2 files changed, 81 insertions(+), 78 deletions(-)
 create mode 100644 frappe/gettext/extractors/utils.py

diff --git a/frappe/gettext/extractors/utils.py b/frappe/gettext/extractors/utils.py
new file mode 100644
index 0000000000..4becbbe64e
--- /dev/null
+++ b/frappe/gettext/extractors/utils.py
@@ -0,0 +1,80 @@
+import re
+
+import frappe
+from frappe.model.utils import InvalidIncludePath, render_include
+
+TRANSLATE_PATTERN = re.compile(
+	r"_\(\s*"  # starts with literal `_(`, ignore following whitespace/newlines
+	# BEGIN: message search
+	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
+	r"(?P<message>((?!\1).)*)"  # Keep matching until string closing identifier is met which is same as 1st capture group
+	r"\1"  # match exact string closing identifier
+	# END: message search
+	# BEGIN: python context search
+	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
+	r"([\"'])"  # start of context string identifier; 5th capture group
+	r"(?P<py_context>((?!\5).)*)"  # capture context string till closing id is found
+	r"\5"  # match context string closure
+	r")?"  # match 0 or 1 context strings
+	# END: python context search
+	# BEGIN: JS context search
+	r"(\s*,\s*(.)*?\s*(,\s*"  # skip message format replacements: ["format", ...] | null | []
+	r"([\"'])"  # start of context string; 11th capture group
+	r"(?P<js_context>((?!\11).)*)"  # capture context string till closing id is found
+	r"\11"  # match context string closure
+	r")*"
+	r")*"  # match one or more context string
+	# END: JS context search
+	r"\s*\)"  # Closing function call ignore leading whitespace/newlines
+)
+
+
+def extract_messages_from_code(code):
+	"""
+	Extracts translatable strings from a code file
+	:param code: code from which translatable files are to be extracted
+	"""
+	from jinja2 import TemplateError
+
+	try:
+		code = frappe.as_unicode(render_include(code))
+
+	# Exception will occur when it encounters John Resig's microtemplating code
+	except (TemplateError, ImportError, InvalidIncludePath, OSError) as e:
+		if isinstance(e, InvalidIncludePath) and hasattr(frappe.local, "message_log"):
+			frappe.clear_last_message()
+
+	messages = []
+
+	for m in TRANSLATE_PATTERN.finditer(code):
+		message = m.group("message")
+		context = m.group("py_context") or m.group("js_context")
+		pos = m.start()
+
+		if is_translatable(message):
+			messages.append([pos, message, context])
+
+	return add_line_number(messages, code)
+
+
+def is_translatable(m):
+	return bool(
+		re.search("[a-zA-Z]", m)
+		and not m.startswith("fa fa-")
+		and not m.endswith("px")
+		and not m.startswith("eval:")
+	)
+
+
+def add_line_number(messages, code):
+	ret = []
+	messages = sorted(messages, key=lambda x: x[0])
+	newlines = [m.start() for m in re.compile(r"\n").finditer(code)]
+	line = 1
+	newline_i = 0
+	for pos, message, context in messages:
+		while newline_i < len(newlines) and pos > newlines[newline_i]:
+			line += 1
+			newline_i += 1
+		ret.append([line, message, context])
+	return ret
diff --git a/frappe/translate.py b/frappe/translate.py
index fc4461e102..7e270eee09 100644
--- a/frappe/translate.py
+++ b/frappe/translate.py
@@ -20,35 +20,11 @@ from csv import reader, writer
 
 import frappe
 from frappe.gettext.extractors.javascript import extract_javascript
+from frappe.gettext.extractors.utils import extract_messages_from_code, is_translatable
 from frappe.gettext.translate import get_translations_from_mo
-from frappe.model.utils import InvalidIncludePath, render_include
 from frappe.query_builder import DocType, Field
 from frappe.utils import cstr, get_bench_path, is_html, strip, strip_html_tags, unique
 
-TRANSLATE_PATTERN = re.compile(
-	r"_\(\s*"  # starts with literal `_(`, ignore following whitespace/newlines
-	# BEGIN: message search
-	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
-	r"(?P<message>((?!\1).)*)"  # Keep matching until string closing identifier is met which is same as 1st capture group
-	r"\1"  # match exact string closing identifier
-	# END: message search
-	# BEGIN: python context search
-	r"(\s*,\s*context\s*=\s*"  # capture `context=` with ignoring whitespace
-	r"([\"'])"  # start of context string identifier; 5th capture group
-	r"(?P<py_context>((?!\5).)*)"  # capture context string till closing id is found
-	r"\5"  # match context string closure
-	r")?"  # match 0 or 1 context strings
-	# END: python context search
-	# BEGIN: JS context search
-	r"(\s*,\s*(.)*?\s*(,\s*"  # skip message format replacements: ["format", ...] | null | []
-	r"([\"'])"  # start of context string; 11th capture group
-	r"(?P<js_context>((?!\11).)*)"  # capture context string till closing id is found
-	r"\11"  # match context string closure
-	r")*"
-	r")*"  # match one or more context string
-	# END: JS context search
-	r"\s*\)"  # Closing function call ignore leading whitespace/newlines
-)
 REPORT_TRANSLATE_PATTERN = re.compile('"([^:,^"]*):')
 CSV_STRIP_WHITESPACE_PATTERN = re.compile(r"{\s?([0-9]+)\s?}")
 
@@ -676,59 +652,6 @@ def extract_messages_from_javascript_code(code: str) -> list[tuple[int, str, str
 	return messages
 
 
-def extract_messages_from_code(code):
-	"""
-	Extracts translatable strings from a code file
-	:param code: code from which translatable files are to be extracted
-	"""
-	from jinja2 import TemplateError
-
-	try:
-		code = frappe.as_unicode(render_include(code))
-
-	# Exception will occur when it encounters John Resig's microtemplating code
-	except (TemplateError, ImportError, InvalidIncludePath, OSError) as e:
-		if isinstance(e, InvalidIncludePath):
-			frappe.clear_last_message()
-
-	messages = []
-
-	for m in TRANSLATE_PATTERN.finditer(code):
-		message = m.group("message")
-		context = m.group("py_context") or m.group("js_context")
-		pos = m.start()
-
-		if is_translatable(message):
-			messages.append([pos, message, context])
-
-	return add_line_number(messages, code)
-
-
-def is_translatable(m):
-	if (
-		re.search("[a-zA-Z]", m)
-		and not m.startswith("fa fa-")
-		and not m.endswith("px")
-		and not m.startswith("eval:")
-	):
-		return True
-	return False
-
-
-def add_line_number(messages, code):
-	ret = []
-	messages = sorted(messages, key=lambda x: x[0])
-	newlines = [m.start() for m in re.compile(r"\n").finditer(code)]
-	line = 1
-	newline_i = 0
-	for pos, message, context in messages:
-		while newline_i < len(newlines) and pos > newlines[newline_i]:
-			line += 1
-			newline_i += 1
-		ret.append([line, message, context])
-	return ret
-
-
 def read_csv_file(path):
 	"""Read CSV file and return as list of list
 

From 10cfcbed676f9005053a656a0aa543425c151089 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Fri, 19 Jan 2024 16:22:54 +0100
Subject: [PATCH 06/64] feat: turn jinja extractor into a generic HTML template
 extractor

... supporting both jinja and JS microtemplates
---
 babel_extractors.csv                       |  2 +-
 frappe/gettext/extractors/html_template.py | 26 ++++++++++++++++++++++
 frappe/gettext/extractors/jinja2.py        | 11 ---------
 3 files changed, 27 insertions(+), 12 deletions(-)
 create mode 100644 frappe/gettext/extractors/html_template.py
 delete mode 100644 frappe/gettext/extractors/jinja2.py

diff --git a/babel_extractors.csv b/babel_extractors.csv
index 133a13e736..37858338ba 100644
--- a/babel_extractors.csv
+++ b/babel_extractors.csv
@@ -6,4 +6,4 @@ hooks.py,frappe.gettext.extractors.navbar.extract
 **/report/*/*.json,frappe.gettext.extractors.report.extract
 **.py,frappe.gettext.extractors.python.extract
 **.js,frappe.gettext.extractors.javascript.extract
-**.html,frappe.gettext.extractors.jinja2.extract
\ No newline at end of file
+**.html,frappe.gettext.extractors.html_template.extract
\ No newline at end of file
diff --git a/frappe/gettext/extractors/html_template.py b/frappe/gettext/extractors/html_template.py
new file mode 100644
index 0000000000..34f51e4032
--- /dev/null
+++ b/frappe/gettext/extractors/html_template.py
@@ -0,0 +1,26 @@
+from jinja2.ext import babel_extract
+
+from .utils import extract_messages_from_code
+
+
+def extract(*args, **kwargs):
+	"""Extract messages from Jinja and JS microtemplates.
+
+	Reuse the babel_extract function from jinja2.ext, but handle our own implementation of `_()`.
+	To handle JS microtemplates, parse all code again using regex."""
+	fileobj = args[0] or kwargs["fileobj"]
+	print(fileobj.name)
+	code = fileobj.read().decode("utf-8")
+
+	for lineno, funcname, messages, comments in babel_extract(*args, **kwargs):
+		if funcname == "_" and isinstance(messages, tuple) and len(messages) > 1:
+			funcname = "pgettext"
+			messages = (messages[-1], messages[0])  # (context, message)
+
+		yield lineno, funcname, messages, comments
+
+	for lineno, message, context in extract_messages_from_code(code):
+		if context:
+			yield lineno, "pgettext", (context, message), []
+		else:
+			yield lineno, "_", message, []
diff --git a/frappe/gettext/extractors/jinja2.py b/frappe/gettext/extractors/jinja2.py
deleted file mode 100644
index ee07ac6cee..0000000000
--- a/frappe/gettext/extractors/jinja2.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from jinja2.ext import babel_extract
-
-
-def extract(*args, **kwargs):
-	"""Reuse the babel_extract function from jinja2.ext, but handle our own implementation of `_()`"""
-	for lineno, funcname, messages, comments in babel_extract(*args, **kwargs):
-		if funcname == "_" and isinstance(messages, tuple) and len(messages) > 1:
-			funcname = "pgettext"
-			messages = (messages[-1], messages[0])  # (context, message)
-
-		yield lineno, funcname, messages, comments

From ef1ee77c99136c7dbc612c13f09c0e4d94e74e4a Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Sat, 20 Jan 2024 07:30:44 +0100
Subject: [PATCH 07/64] test: fully spec test command line

---
 frappe/tests/test_commands.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py
index 7eff954054..fc39adc9a3 100644
--- a/frappe/tests/test_commands.py
+++ b/frappe/tests/test_commands.py
@@ -173,14 +173,18 @@ class BaseTestCommands(FrappeTestCase):
 		cmd_config = {
 			"test_site": TEST_SITE,
 			"admin_password": frappe.conf.admin_password,
-			"root_login": frappe.conf.root_login,
-			"root_password": frappe.conf.root_password,
 			"db_type": frappe.conf.db_type,
+			"db_root_username": frappe.conf.root_login,
+			"db_root_password": frappe.conf.root_password,
 		}
 
 		if not os.path.exists(os.path.join(TEST_SITE, "site_config.json")):
 			cls.execute(
-				"bench new-site {test_site} --admin-password {admin_password} --db-type" " {db_type}",
+				"bench new-site {test_site} "
+				"--admin-password {admin_password} "
+				"--db-root-username {db_root_username} "
+				"--db-root-password {db_root_password} "
+				"--db-type {db_type}",
 				cmd_config,
 			)
 

From f1bcdabb6ce46ede102129fef6b37b5e9248170f Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Sun, 21 Jan 2024 18:24:18 +0100
Subject: [PATCH 08/64] fix: assert minimum contract for frappe.connect

---
 frappe/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index e930b8d500..51ee459b4c 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -301,6 +301,10 @@ def connect(
 			"Instead, explicitly invoke frappe.init(site) with the right config prior to calling frappe.connect(), if necessary."
 		)
 
+	assert db_name or local.conf.db_user, "site must be fully initialized, db_user missing"
+	assert db_name or local.conf.db_name, "site must be fully initialized, db_name missing"
+	assert local.conf.db_password, "site must be fully initialized, db_password missing"
+
 	local.db = get_db(
 		host=local.conf.db_host,
 		port=local.conf.db_port,

From 78aad52b2c1e9af012444a95dda4949aad9f58c8 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Sun, 21 Jan 2024 17:36:22 +0100
Subject: [PATCH 09/64] fix: default db_user where config defaults are set

---
 frappe/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 51ee459b4c..c33cb64f08 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -269,10 +269,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False)
 
 	local.initialised = True
 
-	# Set the user as database name if not set in config
-	if local.conf and local.conf.db_name is not None and local.conf.db_user is None:
-		local.conf.db_user = local.conf.db_name
-
 
 def connect(
 	site: str | None = None, db_name: str | None = None, set_admin_as_user: bool = True
@@ -398,6 +394,9 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 		os.environ.get("FRAPPE_DB_PORT") or config.get("db_port") or db_default_ports(config["db_type"])
 	)
 
+	# Set the user as database name if not set in config
+	config["db_user"] = config.get("db_user") or config.get("db_name")
+
 	return config
 
 

From ee50e6c534e7848de39d70de8d8f71badbfd6ca1 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Sun, 21 Jan 2024 20:19:26 +0100
Subject: [PATCH 10/64] fix: run-tests --module-def as not all doctypes are
 guaranteed to have test_*.py files

---
 frappe/test_runner.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/frappe/test_runner.py b/frappe/test_runner.py
index 35911269cb..635c2a9c6a 100644
--- a/frappe/test_runner.py
+++ b/frappe/test_runner.py
@@ -89,9 +89,22 @@ def main(
 				doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output
 			)
 		elif module_def:
-			doctypes = frappe.db.get_list(
-				"DocType", filters={"module": module_def, "istable": 0}, pluck="name"
+			doctypes = []
+			doctypes_ = frappe.get_list(
+				"DocType",
+				filters={"module": module_def, "istable": 0},
+				fields=["name", "module"],
+				as_list=True,
 			)
+			for doctype, module in doctypes_:
+				test_module = get_module_name(doctype, module, "test_", app=app)
+				try:
+					importlib.import_module(test_module)
+				except Exception:
+					pass
+				else:
+					doctypes.append(doctype)
+
 			ret = run_tests_for_doctype(
 				doctypes, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output
 			)

From 62fa5bc8af2107d7661d588ba884fac2c7694b4b Mon Sep 17 00:00:00 2001
From: marination <maricadsouza221197@gmail.com>
Date: Mon, 22 Jan 2024 16:15:38 +0100
Subject: [PATCH 11/64] fix: Translate Letterhead information string

---
 frappe/printing/doctype/letter_head/letter_head.js | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/frappe/printing/doctype/letter_head/letter_head.js b/frappe/printing/doctype/letter_head/letter_head.js
index a1ef08bce9..7055b5f78a 100644
--- a/frappe/printing/doctype/letter_head/letter_head.js
+++ b/frappe/printing/doctype/letter_head/letter_head.js
@@ -27,22 +27,24 @@ frappe.ui.form.on("Letter Head", {
 	},
 });
 
-const INSTRUCTIONS = `<h4>Letter Head Scripts</h4>
-<p>Header/Footer scripts can be used to add dynamic behaviours.</p>
+const INSTRUCTIONS = `<h4>${__("Letter Head Scripts")}</h4>
+<p>${__("Header/Footer scripts can be used to add dynamic behaviours.")}</p>
 <pre>
 <code>
-// The following Header Script will add the current date to an element in 'Header HTML' with class "header-content"
+// ${__(
+	"The following Header Script will add the current date to an element in 'Header HTML' with class 'header-content'"
+)}
 var el = document.getElementsByClassName("header-content");
 if (el.length > 0) {
 	el[0].textContent += " " + new Date().toGMTString();
 }
 </code>
 </pre>
-<p>You can also access wkhtmltopdf variables (valid only in PDF print):</p>
+<p>${__("You can also access wkhtmltopdf variables (valid only in PDF print):")}</p>
 <pre>
 <code>
-// Get Header and Footer wkhtmltopdf variables
-// Snippet and more variables: https://wkhtmltopdf.org/usage/wkhtmltopdf.txt
+// ${__("Get Header and Footer wkhtmltopdf variables")}
+// ${__("Snippet and more variables:  {0}", ["https://wkhtmltopdf.org/usage/wkhtmltopdf.txt"])}
 var vars = {};
 var query_strings_from_url = document.location.search.substring(1).split('&');
 for (var query_string in query_strings_from_url) {

From f77b99396e03372c08e576952b92632b4ede6a5f Mon Sep 17 00:00:00 2001
From: marination <maricadsouza221197@gmail.com>
Date: Mon, 22 Jan 2024 22:27:08 +0100
Subject: [PATCH 12/64] fix: Custom Script runs twice

- `frappe.get_print` calls `get_response_content("printview")` which calls `www/printview` that adds header and footer scripts
- Now it gets added again via `pdf_header_footer.html` which causes the script to run twice
- Header and Footer html are also added twice but then removed and handled in pdf.py (because printview + wkhtml processing)
- Misc: Use walrus operator
- Remove passed `letterhead` to insert script via pdf.py, printview handles it
---
 frappe/__init__.py                            |  6 ++--
 .../print_formats/pdf_header_footer.html      | 16 +---------
 frappe/utils/pdf.py                           | 29 +++++++------------
 3 files changed, 14 insertions(+), 37 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 7c41c21b51..3d31adbcfa 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -2116,9 +2116,7 @@ def get_print(
 		pdf_options["password"] = password
 
 	html = get_response_content("printview")
-	return (
-		get_pdf(html, options=pdf_options, output=output, letterhead=letterhead) if as_pdf else html
-	)
+	return get_pdf(html, options=pdf_options, output=output) if as_pdf else html
 
 
 def attach_print(
@@ -2157,7 +2155,7 @@ def attach_print(
 			ext = ".pdf"
 			kwargs["as_pdf"] = True
 			content = (
-				get_pdf(html, options={"password": password} if password else None, letterhead=letterhead)
+				get_pdf(html, options={"password": password} if password else None)
 				if html
 				else get_print(doctype, name, **kwargs)
 			)
diff --git a/frappe/templates/print_formats/pdf_header_footer.html b/frappe/templates/print_formats/pdf_header_footer.html
index 91973efb70..189cf43cd9 100644
--- a/frappe/templates/print_formats/pdf_header_footer.html
+++ b/frappe/templates/print_formats/pdf_header_footer.html
@@ -56,28 +56,14 @@
 					}
 				}
 			}
-
-			function custom_load() {
-				if (window.custom_header_footer_script) {
-					window.custom_header_footer_script();
-				}
-			}
 		</script>
 
-		{% if custom_header_footer_script -%}
-		<script>
-			window.custom_header_footer_script = function() {
-				{{ custom_header_footer_script }}
-			}
-		</script>
-		{%- endif %}
-
 		{% for tag in styles -%}
 			{{ tag | string }}
 		{%- endfor %}
 
 	</head>
-	<body onload="subst(); custom_load();">
+	<body onload="subst()">
 		<div class="print-format">
 			<div class="wrapper">
 				{% for tag in content -%}
diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py
index 1f766e4e66..321c65a73c 100644
--- a/frappe/utils/pdf.py
+++ b/frappe/utils/pdf.py
@@ -24,7 +24,7 @@ PDF_CONTENT_ERRORS = [
 ]
 
 
-def pdf_header_html(soup, head, content, styles, html_id, css, script=None):
+def pdf_header_html(soup, head, content, styles, html_id, css):
 	return frappe.render_template(
 		"templates/print_formats/pdf_header_footer.html",
 		{
@@ -35,7 +35,6 @@ def pdf_header_html(soup, head, content, styles, html_id, css, script=None):
 			"css": css,
 			"lang": frappe.local.lang,
 			"layout_direction": "rtl" if is_rtl() else "ltr",
-			"custom_header_footer_script": script,
 		},
 	)
 
@@ -67,15 +66,15 @@ def _guess_template_error_line_number(template) -> int | None:
 				return frame.lineno
 
 
-def pdf_footer_html(soup, head, content, styles, html_id, css, script=None):
+def pdf_footer_html(soup, head, content, styles, html_id, css):
 	return pdf_header_html(
-		soup=soup, head=head, content=content, styles=styles, html_id=html_id, css=css, script=script
+		soup=soup, head=head, content=content, styles=styles, html_id=html_id, css=css
 	)
 
 
-def get_pdf(html, options=None, output: PdfWriter | None = None, letterhead: str | None = None):
+def get_pdf(html, options=None, output: PdfWriter | None = None):
 	html = scrub_urls(html)
-	html, options = prepare_options(html, options, letterhead=letterhead)
+	html, options = prepare_options(html, options)
 
 	options.update({"disable-javascript": "", "disable-local-file-access": ""})
 
@@ -134,7 +133,7 @@ def get_file_data_from_writer(writer_obj):
 	return stream.read()
 
 
-def prepare_options(html, options, letterhead: str | None = None):
+def prepare_options(html, options):
 	if not options:
 		options = {}
 
@@ -156,7 +155,7 @@ def prepare_options(html, options, letterhead: str | None = None):
 	if not options.get("margin-left"):
 		options["margin-left"] = "15mm"
 
-	html, html_options = read_options_from_html(html, letterhead=letterhead)
+	html, html_options = read_options_from_html(html)
 	options.update(html_options or {})
 
 	# cookies
@@ -197,11 +196,11 @@ def get_cookie_options():
 	return options
 
 
-def read_options_from_html(html, letterhead: str | None = None):
+def read_options_from_html(html):
 	options = {}
 	soup = BeautifulSoup(html, "html5lib")
 
-	options.update(prepare_header_footer(soup, letterhead=letterhead))
+	options.update(prepare_header_footer(soup))
 
 	toggle_visible_pdf(soup)
 
@@ -228,7 +227,7 @@ def read_options_from_html(html, letterhead: str | None = None):
 	return str(soup), options
 
 
-def prepare_header_footer(soup: BeautifulSoup, letterhead: str | None = None):
+def prepare_header_footer(soup: BeautifulSoup):
 	options = {}
 
 	head = soup.find("head").contents
@@ -239,13 +238,8 @@ def prepare_header_footer(soup: BeautifulSoup, letterhead: str | None = None):
 
 	# extract header and footer
 	for html_id in ("header-html", "footer-html"):
-		content = soup.find(id=html_id)
-		if content:
-			field = "header_script" if html_id == "header-html" else "footer_script"
-			script = frappe.db.get_value("Letter Head", letterhead, field) if letterhead else None
-
+		if content := soup.find(id=html_id):
 			toggle_visible_pdf(content)
-
 			id_map = {"header-html": "pdf_header_html", "footer-html": "pdf_footer_html"}
 			hook_func = frappe.get_hooks(id_map.get(html_id))
 			html = frappe.get_attr(hook_func[-1])(
@@ -255,7 +249,6 @@ def prepare_header_footer(soup: BeautifulSoup, letterhead: str | None = None):
 				styles=styles,
 				html_id=html_id,
 				css=css,
-				script=script,
 			)
 
 			# there could be multiple instances of header-html/footer-html

From 25168b858fa4b9fa5913898b2c49153d7ecc3f11 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Wed, 24 Jan 2024 11:39:50 +0100
Subject: [PATCH 13/64] feat: db_user can be set by env varbiable

Co-authored-by: Akhil Narang <me@akhilnarang.dev>
---
 frappe/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index c33cb64f08..65591afa89 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -395,7 +395,9 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 	)
 
 	# Set the user as database name if not set in config
-	config["db_user"] = config.get("db_user") or config.get("db_name")
+	config["db_user"] = (
+		os.environ.get("FRAPPE_DB_USER") or config.get("db_user") or config.get("db_name")
+	)
 
 	return config
 

From e7284be5a343d30456565417b74e05f02cd47154 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Sun, 21 Jan 2024 18:21:25 +0100
Subject: [PATCH 14/64] fix: remove db name 'hack'; now the code path is clean

---
 frappe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 65591afa89..39ebed222d 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -304,7 +304,7 @@ def connect(
 	local.db = get_db(
 		host=local.conf.db_host,
 		port=local.conf.db_port,
-		user=local.conf.db_user or db_name or local.conf.db_name,
+		user=db_name or local.conf.db_user,
 		password=local.conf.db_password,
 		cur_db_name=db_name or local.conf.db_name,
 	)

From c57bc94eadd143c6bb8994f0ef6715213c1d2d8f Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Sat, 20 Jan 2024 07:36:35 +0100
Subject: [PATCH 15/64] chore: cleanup frappe.connect invocations

---
 frappe/commands/utils.py                    |  3 +--
 frappe/desk/form/test_form.py               |  7 -------
 frappe/installer.py                         |  5 +++--
 frappe/integrations/offsite_backup_utils.py |  3 ---
 frappe/test_runner.py                       | 11 +++++------
 frappe/tests/test_fmt_money.py              |  7 -------
 frappe/translate.py                         |  6 ------
 7 files changed, 9 insertions(+), 33 deletions(-)

diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py
index a839566130..a6eedf102a 100644
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@@ -772,12 +772,11 @@ def run_tests(
 			click.secho(f"bench --site {site} set-config allow_tests true", fg="green")
 			return
 
-		frappe.init(site=site)
-
 		frappe.flags.skip_before_tests = skip_before_tests
 		frappe.flags.skip_test_records = skip_test_records
 
 		ret = frappe.test_runner.main(
+			site,
 			app,
 			module,
 			doctype,
diff --git a/frappe/desk/form/test_form.py b/frappe/desk/form/test_form.py
index f256b03d27..5028fedf96 100644
--- a/frappe/desk/form/test_form.py
+++ b/frappe/desk/form/test_form.py
@@ -11,10 +11,3 @@ class TestForm(FrappeTestCase):
 		results = get_linked_docs("Role", "System Manager", linkinfo=get_linked_doctypes("Role"))
 		self.assertTrue("User" in results)
 		self.assertTrue("DocType" in results)
-
-
-if __name__ == "__main__":
-	import unittest
-
-	frappe.connect()
-	unittest.main()
diff --git a/frappe/installer.py b/frappe/installer.py
index fce104ad99..68fd4d87d2 100644
--- a/frappe/installer.py
+++ b/frappe/installer.py
@@ -20,9 +20,10 @@ from frappe.utils.dashboard import sync_dashboards
 from frappe.utils.synchronization import filelock
 
 
-def _is_scheduler_enabled() -> bool:
+def _is_scheduler_enabled(site) -> bool:
 	enable_scheduler = False
 	try:
+		frappe.init(site=site)
 		frappe.connect()
 		enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler"))
 	except Exception:
@@ -78,7 +79,7 @@ def _new_site(
 
 	try:
 		# enable scheduler post install?
-		enable_scheduler = _is_scheduler_enabled()
+		enable_scheduler = _is_scheduler_enabled(site)
 	except Exception:
 		enable_scheduler = False
 
diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index e2e2fed40f..a71fe0e28d 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -41,9 +41,6 @@ def send_email(success, service_name, doctype, email_field, error_status=None):
 
 
 def get_recipients(doctype, email_field):
-	if not frappe.db:
-		frappe.connect()
-
 	return split_emails(frappe.db.get_value(doctype, None, email_field))
 
 
diff --git a/frappe/test_runner.py b/frappe/test_runner.py
index 35911269cb..0b81aa75bd 100644
--- a/frappe/test_runner.py
+++ b/frappe/test_runner.py
@@ -38,6 +38,7 @@ def xmlrunner_wrapper(output):
 
 
 def main(
+	site=None,
 	app=None,
 	module=None,
 	doctype=None,
@@ -53,6 +54,10 @@ def main(
 ):
 	global unittest_runner
 
+	frappe.init(site=site)
+	if not frappe.db:
+		frappe.connect()
+
 	if doctype_list_path:
 		app, doctype_list_path = doctype_list_path.split(os.path.sep, 1)
 		with open(frappe.get_app_path(app, doctype_list_path)) as f:
@@ -69,9 +74,6 @@ def main(
 		frappe.flags.print_messages = verbose
 		frappe.flags.in_test = True
 
-		if not frappe.db:
-			frappe.connect()
-
 		# workaround! since there is no separate test db
 		frappe.clear_cache()
 		scheduler_disabled_by_user = frappe.utils.scheduler.is_scheduler_disabled(verbose=False)
@@ -329,9 +331,6 @@ def _add_test(app, path, filename, verbose, test_suite=None):
 
 
 def make_test_records(doctype, verbose=0, force=False, commit=False):
-	if not frappe.db:
-		frappe.connect()
-
 	if frappe.flags.skip_test_records:
 		return
 
diff --git a/frappe/tests/test_fmt_money.py b/frappe/tests/test_fmt_money.py
index fff5011189..0fbd38cbcc 100644
--- a/frappe/tests/test_fmt_money.py
+++ b/frappe/tests/test_fmt_money.py
@@ -100,10 +100,3 @@ class TestFmtMoney(FrappeTestCase):
 		frappe.db.set_value("Currency", "JPY", "symbol_on_right", 1)
 		self.assertEqual(fmt_money(100.0, format="#,###.##", currency="JPY"), "100.00 ¥")
 		self.assertEqual(fmt_money(100.0, format="#,###.##", currency="USD"), "$ 100.00")
-
-
-if __name__ == "__main__":
-	import unittest
-
-	frappe.connect()
-	unittest.main()
diff --git a/frappe/translate.py b/frappe/translate.py
index fc4461e102..2e08494df7 100644
--- a/frappe/translate.py
+++ b/frappe/translate.py
@@ -239,9 +239,6 @@ def get_translation_dict_from_file(path, lang, app, throw=False) -> dict[str, st
 
 
 def get_user_translations(lang):
-	if not frappe.db:
-		frappe.connect()
-
 	def _read_from_db():
 		user_translations = {}
 		translations = frappe.get_all(
@@ -1054,9 +1051,6 @@ def get_all_languages(with_language_name: bool = False) -> list:
 	def get_all_language_with_name():
 		return frappe.get_all("Language", ["language_code", "language_name"], {"enabled": 1})
 
-	if not frappe.db:
-		frappe.connect()
-
 	if with_language_name:
 		return frappe.cache.get_value("languages_with_name", get_all_language_with_name)
 	else:

From 0c6c0c5f50ed894082b8cb37d9cee919f9fa2511 Mon Sep 17 00:00:00 2001
From: Govind Jangid <govindjangidsj@gmail.com>
Date: Mon, 29 Jan 2024 12:17:58 +0530
Subject: [PATCH 16/64] feat: use_first_day_of_period

---
 .../auto_email_report/auto_email_report.json  | 10 ++-
 .../auto_email_report/auto_email_report.py    | 79 +++++++++++++++----
 2 files changed, 73 insertions(+), 16 deletions(-)

diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.json b/frappe/email/doctype/auto_email_report/auto_email_report.json
index 75a9e99c96..ade2706666 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.json
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.json
@@ -25,6 +25,7 @@
   "to_date_field",
   "column_break_17",
   "dynamic_date_period",
+  "use_first_day_of_period",
   "email_settings",
   "email_to",
   "day_of_week",
@@ -207,10 +208,17 @@
    "fieldtype": "Link",
    "label": "Sender",
    "options": "Email Account"
+  },
+  {
+   "default": "0",
+   "description": "To use From Date based on First Day of Current Period",
+   "fieldname": "use_first_day_of_period",
+   "fieldtype": "Check",
+   "label": "Use First Day of Period"
   }
  ],
  "links": [],
- "modified": "2022-09-08 15:31:55.031023",
+ "modified": "2024-01-29 11:42:27.433958",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Auto Email Report",
diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index 6fe2596d7f..2f9c13d4c0 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -4,6 +4,7 @@
 import calendar
 from datetime import timedelta
 from email.utils import formataddr
+import datetime
 
 import frappe
 from frappe import _
@@ -21,6 +22,11 @@ from frappe.utils import (
 	now_datetime,
 	today,
 	validate_email_address,
+	get_first_day_of_week,
+	get_first_day,
+	get_quarter_start,
+	get_year_start,
+	getdate,
 )
 from frappe.utils.csvutils import to_csv
 from frappe.utils.xlsxutils import make_xlsx
@@ -36,13 +42,9 @@ class AutoEmailReport(Document):
 		from frappe.types import DF
 
 		data_modified_till: DF.Int
-		day_of_week: DF.Literal[
-			"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"
-		]
+		day_of_week: DF.Literal["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
 		description: DF.TextEditor | None
-		dynamic_date_period: DF.Literal[
-			"", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"
-		]
+		dynamic_date_period: DF.Literal["", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"]
 		email_to: DF.SmallText
 		enabled: DF.Check
 		filter_meta: DF.Text | None
@@ -57,8 +59,10 @@ class AutoEmailReport(Document):
 		send_if_data: DF.Check
 		sender: DF.Link | None
 		to_date_field: DF.Literal
+		use_first_day_of_period: DF.Check
 		user: DF.Link
 	# end: auto-generated types
+
 	def autoname(self):
 		self.name = _(self.report)
 		if frappe.db.exists("Auto Email Report", self.name):
@@ -207,17 +211,45 @@ class AutoEmailReport(Document):
 		self.filters = frappe.parse_json(self.filters)
 
 		to_date = today()
-		from_date_value = {
-			"Daily": ("days", -1),
-			"Weekly": ("weeks", -1),
-			"Monthly": ("months", -1),
-			"Quarterly": ("months", -3),
-			"Half Yearly": ("months", -6),
-			"Yearly": ("years", -1),
-		}[self.dynamic_date_period]
+		
+		if self.use_first_day_of_period:
+			from_date_value = {
+				"Daily": ("days", -1),
+				"Weekly": ("weeks", 0),
+				"Monthly": ("months", 0),
+				"Quarterly": ("months", 0),
+				"Half Yearly": ("months", 0),
+				"Yearly": ("years", 0),
+			}[self.dynamic_date_period]
 
-		from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
+			from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
+			
+			if self.dynamic_date_period == "Weekly":
+				from_date = get_first_day_of_week(from_date)
+			elif self.dynamic_date_period == "Monthly":
+				from_date = get_first_day(from_date)
+			elif self.dynamic_date_period == "Quarterly":
+				from_date = get_quarter_start(from_date)
+			elif self.dynamic_date_period == "Half Yearly":
+				from_date = get_half_year_start(from_date)
+			elif self.dynamic_date_period == "Yearly":
+				from_date = get_year_start(from_date)
 
+			self.set_date_filters(from_date, to_date)
+		else:
+			from_date_value = {
+				"Daily": ("days", -1),
+				"Weekly": ("weeks", -1),
+				"Monthly": ("months", -1),
+				"Quarterly": ("months", -3),
+				"Half Yearly": ("months", -6),
+				"Yearly": ("years", -1),
+			}[self.dynamic_date_period]
+
+			from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
+			self.set_date_filters(from_date, to_date)
+		
+	def set_date_filters(self, from_date, to_date):
 		self.filters[self.from_date_field] = from_date
 		self.filters[self.to_date_field] = to_date
 
@@ -332,3 +364,20 @@ def update_field_types(columns):
 			col.fieldtype = "Data"
 			col.options = ""
 	return columns
+
+DATE_FORMAT = "%Y-%m-%d"
+def get_half_year_start(as_str=False):
+    """
+    Returns the first day of the current half-year based on the current date.
+    """
+    today_date = getdate(today())
+
+    half_year = 1 if today_date.month <= 6 else 2
+
+    year = today_date.year if half_year == 1 else today_date.year + 1
+    month = 1 if half_year == 1 else 7
+    day = 1
+
+    result_date = datetime.date(year, month, day)
+
+    return result_date if not as_str else result_date.strftime(DATE_FORMAT)
\ No newline at end of file

From 503222391bbf673986c13fe8b15e8d804d95c136 Mon Sep 17 00:00:00 2001
From: Govind Jangid <63184660+Govind-Jangid@users.noreply.github.com>
Date: Mon, 29 Jan 2024 12:30:35 +0530
Subject: [PATCH 17/64] fix: formatting

---
 .../doctype/auto_email_report/auto_email_report.py     | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index 2f9c13d4c0..fa939f9815 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -42,9 +42,13 @@ class AutoEmailReport(Document):
 		from frappe.types import DF
 
 		data_modified_till: DF.Int
-		day_of_week: DF.Literal["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
+		day_of_week: DF.Literal[
++			"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"
++		]
 		description: DF.TextEditor | None
-		dynamic_date_period: DF.Literal["", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"]
+		dynamic_date_period: DF.Literal[
++			"", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"
++		]
 		email_to: DF.SmallText
 		enabled: DF.Check
 		filter_meta: DF.Text | None
@@ -380,4 +384,4 @@ def get_half_year_start(as_str=False):
 
     result_date = datetime.date(year, month, day)
 
-    return result_date if not as_str else result_date.strftime(DATE_FORMAT)
\ No newline at end of file
+    return result_date if not as_str else result_date.strftime(DATE_FORMAT)

From bb746d5e9c60eda4d1786c29d653410b0c2ed68d Mon Sep 17 00:00:00 2001
From: Govind Jangid <63184660+Govind-Jangid@users.noreply.github.com>
Date: Mon, 29 Jan 2024 12:31:54 +0530
Subject: [PATCH 18/64] fix: formatting

---
 .../email/doctype/auto_email_report/auto_email_report.py  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index fa939f9815..6818607a49 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -43,12 +43,12 @@ class AutoEmailReport(Document):
 
 		data_modified_till: DF.Int
 		day_of_week: DF.Literal[
-+			"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"
-+		]
+			"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"
+		]
 		description: DF.TextEditor | None
 		dynamic_date_period: DF.Literal[
-+			"", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"
-+		]
+			"", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"
+		]
 		email_to: DF.SmallText
 		enabled: DF.Check
 		filter_meta: DF.Text | None

From 9e74bfbb8c997c37fe70bf8516cc6d96dbcee027 Mon Sep 17 00:00:00 2001
From: Govind Jangid <63184660+Govind-Jangid@users.noreply.github.com>
Date: Mon, 29 Jan 2024 14:35:54 +0530
Subject: [PATCH 19/64] fix: description

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/email/doctype/auto_email_report/auto_email_report.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.json b/frappe/email/doctype/auto_email_report/auto_email_report.json
index ade2706666..714c5e50e0 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.json
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.json
@@ -211,7 +211,7 @@
   },
   {
    "default": "0",
-   "description": "To use From Date based on First Day of Current Period",
+To begin the date range at the start of the chosen period. For example, if 'Year' is selected as the period, the report will start from January 1st of the current year.
    "fieldname": "use_first_day_of_period",
    "fieldtype": "Check",
    "label": "Use First Day of Period"

From 39d3a5e8f47147b6ac3cdfa8ed9dccc6e81f6257 Mon Sep 17 00:00:00 2001
From: Govind Jangid <63184660+Govind-Jangid@users.noreply.github.com>
Date: Mon, 29 Jan 2024 14:39:58 +0530
Subject: [PATCH 20/64] fix: suggestion

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 .../auto_email_report/auto_email_report.py       | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index 6818607a49..2b82719232 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -217,18 +217,10 @@ class AutoEmailReport(Document):
 		to_date = today()
 		
 		if self.use_first_day_of_period:
-			from_date_value = {
-				"Daily": ("days", -1),
-				"Weekly": ("weeks", 0),
-				"Monthly": ("months", 0),
-				"Quarterly": ("months", 0),
-				"Half Yearly": ("months", 0),
-				"Yearly": ("years", 0),
-			}[self.dynamic_date_period]
-
-			from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
-			
-			if self.dynamic_date_period == "Weekly":
+			from_date = to_date
+			if self.dynamic_date_period == "Daily":
+				from_date = add_to_date(to_date, days=-1)
+			elif self.dynamic_date_period == "Weekly":
 				from_date = get_first_day_of_week(from_date)
 			elif self.dynamic_date_period == "Monthly":
 				from_date = get_first_day(from_date)

From cb841bde6d05fa8befd87fd278ef6a51c00ef409 Mon Sep 17 00:00:00 2001
From: Govind Jangid <63184660+Govind-Jangid@users.noreply.github.com>
Date: Mon, 29 Jan 2024 14:47:42 +0530
Subject: [PATCH 21/64] fix: depends on

---
 .../email/doctype/auto_email_report/auto_email_report.json   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.json b/frappe/email/doctype/auto_email_report/auto_email_report.json
index 714c5e50e0..f208626500 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.json
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.json
@@ -211,9 +211,10 @@
   },
   {
    "default": "0",
-To begin the date range at the start of the chosen period. For example, if 'Year' is selected as the period, the report will start from January 1st of the current year.
+   "description": "To begin the date range at the start of the chosen period. For example, if 'Year' is selected as the period, the report will start from January 1st of the current year.",
    "fieldname": "use_first_day_of_period",
    "fieldtype": "Check",
+   "depends_on": "eval: doc.dynamic_date_period != 'Daily'",
    "label": "Use First Day of Period"
   }
  ],
@@ -253,4 +254,4 @@ To begin the date range at the start of the chosen period. For example, if 'Year
  "sort_order": "DESC",
  "states": [],
  "track_changes": 1
-}
\ No newline at end of file
+}

From bcca1da8d841e985cbe304cf9cbb99e73475bbe4 Mon Sep 17 00:00:00 2001
From: marination <maricadsouza221197@gmail.com>
Date: Mon, 29 Jan 2024 13:25:45 +0100
Subject: [PATCH 22/64] fix: Extract header/footer html into `content` to
 simplify things

- Extract header/footer html into `content` first, then remove any instances from the main content since it is already rendered via the extracted header/footer html
---
 frappe/utils/pdf.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py
index 321c65a73c..be25b8cd48 100644
--- a/frappe/utils/pdf.py
+++ b/frappe/utils/pdf.py
@@ -238,7 +238,13 @@ def prepare_header_footer(soup: BeautifulSoup):
 
 	# extract header and footer
 	for html_id in ("header-html", "footer-html"):
-		if content := soup.find(id=html_id):
+		if content := soup.find(id=html_id).extract():
+			# `header/footer-html` are extracted, rendered as html
+			# and passed in wkhtmltopdf options (as '--header/footer-html')
+			# Remove instances of them from main content for render_template
+			for tag in soup.find_all(id=html_id):
+				tag.extract()
+
 			toggle_visible_pdf(content)
 			id_map = {"header-html": "pdf_header_html", "footer-html": "pdf_footer_html"}
 			hook_func = frappe.get_hooks(id_map.get(html_id))
@@ -251,10 +257,6 @@ def prepare_header_footer(soup: BeautifulSoup):
 				css=css,
 			)
 
-			# there could be multiple instances of header-html/footer-html
-			for tag in soup.find_all(id=html_id):
-				tag.extract()
-
 			# create temp file
 			fname = os.path.join("/tmp", f"frappe-pdf-{frappe.generate_hash()}.html")
 			with open(fname, "wb") as f:

From b59ba82ae64f3eb81a89d7d7554d9d04b430c823 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Tue, 30 Jan 2024 11:48:19 +0100
Subject: [PATCH 23/64] fix(Data Import): translatability

---
 .../core/doctype/data_import/data_import.js   | 43 ++++++++-----------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js
index 9f0e11a7b7..06797a0b40 100644
--- a/frappe/core/doctype/data_import/data_import.js
+++ b/frappe/core/doctype/data_import/data_import.js
@@ -135,34 +135,29 @@ frappe.ui.form.on("Data Import", {
 				let failed_records = cint(r.message.failed);
 				let total_records = cint(r.message.total_records);
 
-				if (!total_records) return;
-				let action, message;
-				if (frm.doc.import_type === "Insert New Records") {
-					action = "imported";
-				} else {
-					action = "updated";
+				if (!total_records) {
+					return;
 				}
 
-				if (failed_records === 0) {
-					let message_args = [action, successful_records];
-					if (successful_records === 1) {
-						message = __("Successfully {0} 1 record.", message_args);
-					} else {
-						message = __("Successfully {0} {1} records.", message_args);
-					}
+				let message;
+				if (frm.doc.import_type === "Insert New Records") {
+					message = __("Successfully imported {0} out of {1} records.", [
+						successful_records,
+						total_records,
+					]);
 				} else {
-					let message_args = [action, successful_records, total_records];
-					if (successful_records === 1) {
-						message = __(
-							"Successfully {0} {1} record out of {2}. Click on Export Errored Rows, fix the errors and import again.",
-							message_args
+					message = __("Successfully updated {0} out of {1} records.", [
+						successful_records,
+						total_records,
+					]);
+				}
+
+				if (failed_records > 0) {
+					message +=
+						"<br/>" +
+						__(
+							"Please click on 'Export Errored Rows', fix the errors and import again."
 						);
-					} else {
-						message = __(
-							"Successfully {0} {1} records out of {2}. Click on Export Errored Rows, fix the errors and import again.",
-							message_args
-						);
-					}
 				}
 
 				// If the job timed out, display an extra hint

From ba64285c66e12b2837f66d723d5d857a36cc2565 Mon Sep 17 00:00:00 2001
From: Akhil Narang <me@akhilnarang.dev>
Date: Tue, 30 Jan 2024 17:07:47 +0530
Subject: [PATCH 24/64] Revert "fix(data_import): respect the value of
 show_failed_logs checkbox"

This reverts commit 34f03a2de24a2e34985a1e2e5599e287081f3cf2.
---
 frappe/core/doctype/data_import/data_import.js | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js
index 9f0e11a7b7..a93671e3e3 100644
--- a/frappe/core/doctype/data_import/data_import.js
+++ b/frappe/core/doctype/data_import/data_import.js
@@ -506,13 +506,7 @@ frappe.ui.form.on("Data Import", {
 	},
 
 	show_import_log(frm) {
-		if (!frm.doc.show_failed_logs) {
-			frm.toggle_display("import_log_preview", false);
-			return;
-		}
-
 		frm.toggle_display("import_log_section", false);
-		frm.toggle_display("import_log_preview", true);
 
 		if (frm.import_in_progress) {
 			return;

From db1e692d4f1ea44aab10a065d7cd31e5256fae6f Mon Sep 17 00:00:00 2001
From: Akhil Narang <me@akhilnarang.dev>
Date: Tue, 30 Jan 2024 17:08:14 +0530
Subject: [PATCH 25/64] chore: reword checkbox

Signed-off-by: Akhil Narang <me@akhilnarang.dev>
---
 frappe/core/doctype/data_import/data_import.json | 4 ++--
 frappe/core/doctype/data_import/data_import.py   | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json
index 97716219a2..02d65c8004 100644
--- a/frappe/core/doctype/data_import/data_import.json
+++ b/frappe/core/doctype/data_import/data_import.json
@@ -139,7 +139,7 @@
    "default": "0",
    "fieldname": "show_failed_logs",
    "fieldtype": "Check",
-   "label": "Show Failed Logs"
+   "label": "Show Only Failed Logs"
   },
   {
    "depends_on": "eval:!doc.__islocal && !doc.import_file",
@@ -171,7 +171,7 @@
  ],
  "hide_toolbar": 1,
  "links": [],
- "modified": "2023-12-15 12:45:49.452834",
+ "modified": "2024-01-30 17:08:05.566686",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Data Import",
diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py
index ac28f9091f..c674122f30 100644
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@@ -38,7 +38,6 @@ class DataImport(Document):
 		submit_after_import: DF.Check
 		template_options: DF.Code | None
 		template_warnings: DF.Code | None
-
 	# end: auto-generated types
 
 	def validate(self):

From d838da51ddf3dcc74a883b43567e941eb5ba453e Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Tue, 30 Jan 2024 20:29:19 +0530
Subject: [PATCH 26/64] fix: short circuit private files perm check

For guest users there's no point in checking permissions, they'll eventually fail... instead just fail immediately.
---
 frappe/utils/response.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/utils/response.py b/frappe/utils/response.py
index 9fe10545d1..5be201499b 100644
--- a/frappe/utils/response.py
+++ b/frappe/utils/response.py
@@ -265,6 +265,9 @@ def download_backup(path):
 def download_private_file(path: str) -> Response:
 	"""Checks permissions and sends back private file"""
 
+	if frappe.session.user == "Guest":
+		raise Forbidden(_("You don't have permission to access this file"))
+
 	files = frappe.get_all("File", filters={"file_url": path}, fields="*")
 	# this file might be attached to multiple documents
 	# if the file is accessible from any one of those documents

From 40438b400f90fa2900e6e579156afd9afeac4e03 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Tue, 30 Jan 2024 21:52:11 +0530
Subject: [PATCH 27/64] ci: set node env as production in tests

---
 .github/workflows/server-tests.yml | 2 ++
 .github/workflows/ui-tests.yml     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml
index 39dc99374e..9da7244527 100644
--- a/.github/workflows/server-tests.yml
+++ b/.github/workflows/server-tests.yml
@@ -43,6 +43,8 @@ jobs:
     needs: checkrun
     if: ${{ needs.checkrun.outputs.build == 'strawberry' }}
     timeout-minutes: 60
+    env:
+      NODE_ENV: "production"
 
     strategy:
       fail-fast: false
diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml
index 33a7cde8f4..ecda8cc6b1 100644
--- a/.github/workflows/ui-tests.yml
+++ b/.github/workflows/ui-tests.yml
@@ -42,6 +42,8 @@ jobs:
     needs: checkrun
     if: ${{ needs.checkrun.outputs.build == 'strawberry' && github.repository_owner == 'frappe' }}
     timeout-minutes: 60
+    env:
+      NODE_ENV: "production"
 
     strategy:
       fail-fast: false

From 3abbde328477d1a787a9e4c8b6ccaf3176a31d50 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Tue, 30 Jan 2024 17:49:35 +0100
Subject: [PATCH 28/64] fix: shell escaping on external commands

---
 frappe/database/__init__.py | 10 ----------
 frappe/utils/backups.py     | 14 ++++++++------
 2 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py
index 530bd4c700..7cd2dda4a9 100644
--- a/frappe/database/__init__.py
+++ b/frappe/database/__init__.py
@@ -75,12 +75,7 @@ def get_command(
 		else:
 			bin, bin_name = which("psql"), "psql"
 
-		host = frappe.utils.esc(host, "$ ")
-		user = frappe.utils.esc(user, "$ ")
-		db_name = frappe.utils.esc(db_name, "$ ")
-
 		if password:
-			password = frappe.utils.esc(password, "$ ")
 			conn_string = f"postgresql://{user}:{password}@{host}:{port}/{db_name}"
 		else:
 			conn_string = f"postgresql://{user}@{host}:{port}/{db_name}"
@@ -96,10 +91,6 @@ def get_command(
 		else:
 			bin, bin_name = which("mariadb") or which("mysql"), "mariadb"
 
-		host = frappe.utils.esc(host, "$ ")
-		user = frappe.utils.esc(user, "$ ")
-		db_name = frappe.utils.esc(db_name, "$ ")
-
 		command = [
 			f"--user={user}",
 			f"--host={host}",
@@ -107,7 +98,6 @@ def get_command(
 		]
 
 		if password:
-			password = frappe.utils.esc(password, "$ ")
 			command.append(f"--password={password}")
 
 		if dump:
diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index 295aff2b4d..40aba696f5 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -370,6 +370,8 @@ class BackupGenerator:
 			n.write(c.read())
 
 	def take_dump(self):
+		import shlex
+
 		import frappe.utils
 		from frappe.utils.change_log import get_app_branch
 
@@ -419,15 +421,15 @@ class BackupGenerator:
 		extra = []
 		if self.db_type == "mariadb":
 			if self.backup_includes:
-				extra.extend([f"'{x}'" for x in self.backup_includes])
+				extra.extend(self.backup_includes)
 			elif self.backup_excludes:
-				extra.extend([f"--ignore-table='{self.db_name}.{table}'" for table in self.backup_excludes])
+				extra.extend([f"--ignore-table={self.db_name}.{table}" for table in self.backup_excludes])
 
 		elif self.db_type == "postgres":
 			if self.backup_includes:
-				extra.extend([f"--table='public.\"{table}\"'" for table in self.backup_includes])
+				extra.extend([f'--table=public."{table}"' for table in self.backup_includes])
 			elif self.backup_excludes:
-				extra.extend([f"--exclude-table-data='public.\"{table}\"'" for table in self.backup_excludes])
+				extra.extend([f'--exclude-table-data=public."{table}"' for table in self.backup_excludes])
 
 		from frappe.database import get_command
 
@@ -446,11 +448,11 @@ class BackupGenerator:
 				exc=frappe.ExecutableNotFound,
 			)
 		cmd.append(bin)
-		cmd.extend(args)
+		cmd.append(shlex.join(args))
 
 		command = " ".join(["set -o pipefail;"] + cmd + ["|", gzip_exc, ">>", self.backup_path_db])
 		if self.verbose:
-			print(command.replace(frappe.utils.esc(self.password, "$ "), "*" * 10) + "\n")
+			print(command.replace(shlex.quote(self.password), "*" * 10) + "\n")
 
 		frappe.utils.execute_in_shell(command, low_priority=True, check_exit_code=True)
 

From dedadbb0d908eb660c5935b91051d520b0cff7e2 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Tue, 30 Jan 2024 21:21:05 +0530
Subject: [PATCH 29/64] perf: optional faster perm check for files

---
 frappe/core/doctype/file/file.py |  7 +++++++
 frappe/email/receive.py          |  2 +-
 frappe/tests/test_api.py         | 17 ++++++++++++++++-
 frappe/utils/response.py         |  7 ++++++-
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py
index de4375ae6c..02cf453d2b 100755
--- a/frappe/core/doctype/file/file.py
+++ b/frappe/core/doctype/file/file.py
@@ -756,6 +756,13 @@ class File(Document):
 		self.save_file(content=optimized_content, overwrite=True)
 		self.save()
 
+	@property
+	def unique_url(self) -> str:
+		"""Unique URL contains file ID in URL to speed up permisison checks."""
+		from urllib.parse import urlencode
+
+		return self.file_url + "?" + urlencode({"fid": self.name})
+
 	@staticmethod
 	def zip_files(files):
 		zip_file = io.BytesIO()
diff --git a/frappe/email/receive.py b/frappe/email/receive.py
index 72a2dfce82..74d7f84dac 100644
--- a/frappe/email/receive.py
+++ b/frappe/email/receive.py
@@ -673,7 +673,7 @@ class InboundMail(Email):
 		content = self.content
 		for file in attachments:
 			if file.name in self.cid_map and self.cid_map[file.name]:
-				content = content.replace(f"cid:{self.cid_map[file.name]}", file.file_url)
+				content = content.replace(f"cid:{self.cid_map[file.name]}", file.unique_url)
 		return content
 
 	def is_notification(self):
diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py
index 8d3065982b..8dd68dd644 100644
--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@@ -15,7 +15,7 @@ from werkzeug.test import TestResponse
 import frappe
 from frappe.installer import update_site_config
 from frappe.tests.utils import FrappeTestCase, patch_hooks
-from frappe.utils import cint, get_test_client, get_url
+from frappe.utils import cint, get_site_url, get_test_client, get_url
 
 try:
 	_site = frappe.local.site
@@ -432,6 +432,21 @@ class TestResponse(FrappeAPITestCase):
 		self.assertGreater(cint(response.headers["content-length"]), 0)
 		self.assertEqual(response.headers["content-disposition"], f'filename="{encoded_filename}"')
 
+	def test_download_private_file_with_unique_url(self):
+		test_content = frappe.generate_hash()
+		file = frappe.get_doc(
+			{
+				"doctype": "File",
+				"file_name": test_content,
+				"content": test_content,
+				"is_private": 1,
+			}
+		)
+		file.insert()
+
+		self.assertEqual(self.get(file.unique_url, {"sid": self.sid}).text, test_content)
+		self.assertEqual(self.get(file.file_url, {"sid": self.sid}).text, test_content)
+
 
 def generate_admin_keys():
 	from frappe.core.doctype.user.user import generate_keys
diff --git a/frappe/utils/response.py b/frappe/utils/response.py
index 5be201499b..116f3cd611 100644
--- a/frappe/utils/response.py
+++ b/frappe/utils/response.py
@@ -268,7 +268,12 @@ def download_private_file(path: str) -> Response:
 	if frappe.session.user == "Guest":
 		raise Forbidden(_("You don't have permission to access this file"))
 
-	files = frappe.get_all("File", filters={"file_url": path}, fields="*")
+	filters = {"file_url": path}
+	if frappe.form_dict.fid:
+		filters["name"] = str(frappe.form_dict.fid)
+
+	files = frappe.get_all("File", filters=filters, fields="*")
+
 	# this file might be attached to multiple documents
 	# if the file is accessible from any one of those documents
 	# then it should be downloadable

From 53d6d156ec3d2b9638e045230b4aaa8ff8c1ea49 Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Wed, 31 Jan 2024 12:00:39 +0530
Subject: [PATCH 30/64] chore: cleanup doctype descriptions (#24609)

---
 frappe/email/doctype/newsletter/newsletter.json         | 7 +++----
 frappe/geo/doctype/currency/currency.json               | 4 ++--
 frappe/website/doctype/blog_settings/blog_settings.json | 4 ++--
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json
index e7c902697f..e03cc506e6 100644
--- a/frappe/email/doctype/newsletter/newsletter.json
+++ b/frappe/email/doctype/newsletter/newsletter.json
@@ -3,7 +3,7 @@
  "allow_guest_to_view": 1,
  "allow_rename": 1,
  "creation": "2013-01-10 16:34:31",
- "description": "Create and Send Newsletters",
+ "description": "Create and send emails to a specific group of subscribers periodically.",
  "doctype": "DocType",
  "document_type": "Other",
  "engine": "InnoDB",
@@ -244,8 +244,7 @@
    "fieldname": "campaign",
    "fieldtype": "Link",
    "label": "Campaign",
-   "options": "Marketing Campaign",
-   "reqd": 0
+   "options": "Marketing Campaign"
   }
  ],
  "has_web_view": 1,
@@ -254,7 +253,7 @@
  "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2023-12-29 18:04:13.270523",
+ "modified": "2024-01-30 14:05:50.645802",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",
diff --git a/frappe/geo/doctype/currency/currency.json b/frappe/geo/doctype/currency/currency.json
index 00dfe248c9..9a4df0f117 100644
--- a/frappe/geo/doctype/currency/currency.json
+++ b/frappe/geo/doctype/currency/currency.json
@@ -4,7 +4,7 @@
  "allow_rename": 1,
  "autoname": "field:currency_name",
  "creation": "2013-01-28 10:06:02",
- "description": "**Currency** Master",
+ "description": "Currency list stores the currency value, its symbol and fraction unit",
  "doctype": "DocType",
  "document_type": "Setup",
  "engine": "InnoDB",
@@ -82,7 +82,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2024-01-17 15:37:31.605278",
+ "modified": "2024-01-30 13:18:12.053557",
  "modified_by": "Administrator",
  "module": "Geo",
  "name": "Currency",
diff --git a/frappe/website/doctype/blog_settings/blog_settings.json b/frappe/website/doctype/blog_settings/blog_settings.json
index fe5417f812..95236c8102 100644
--- a/frappe/website/doctype/blog_settings/blog_settings.json
+++ b/frappe/website/doctype/blog_settings/blog_settings.json
@@ -1,7 +1,7 @@
 {
  "actions": [],
  "creation": "2013-03-11 17:48:16",
- "description": "Blog Settings",
+ "description": "Settings to control blog categories and interactions like comments and likes",
  "doctype": "DocType",
  "engine": "InnoDB",
  "field_order": [
@@ -131,7 +131,7 @@
  "idx": 1,
  "issingle": 1,
  "links": [],
- "modified": "2022-10-18 15:01:36.202010",
+ "modified": "2024-01-30 14:13:12.477755",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Settings",

From 161bb4250871d23c346b40f1640562b054089c8b Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Wed, 31 Jan 2024 09:58:56 +0100
Subject: [PATCH 31/64] fix: deprioritize precedence of db_name argument to
 connect

---
 frappe/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 39ebed222d..f5d72fa6ba 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -304,9 +304,9 @@ def connect(
 	local.db = get_db(
 		host=local.conf.db_host,
 		port=local.conf.db_port,
-		user=db_name or local.conf.db_user,
+		user=local.conf.db_user or db_name,
 		password=local.conf.db_password,
-		cur_db_name=db_name or local.conf.db_name,
+		cur_db_name=local.conf.db_name or db_name,
 	)
 	if set_admin_as_user:
 		set_user("Administrator")

From a25e68a763b84f571e60d372a3d0b0ac2e265ff0 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Wed, 31 Jan 2024 14:46:27 +0530
Subject: [PATCH 32/64] fix: make rate_limiter respect multitenancy (#24634)

* fix: make rate_limiter respect multitenancy

* fix: lower rate limit window for password reset

* refactor: Use redis_wrapper
---
 frappe/core/doctype/server_script/test_server_script.py | 1 -
 frappe/core/doctype/user/user.py                        | 2 +-
 frappe/rate_limiter.py                                  | 2 +-
 frappe/utils/password.py                                | 2 +-
 4 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py
index b83d1edda4..f53d69304a 100644
--- a/frappe/core/doctype/server_script/test_server_script.py
+++ b/frappe/core/doctype/server_script/test_server_script.py
@@ -238,7 +238,6 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
 		script.execute_method()
 
 	def test_server_script_rate_limiting(self):
-		# why not
 		script1 = frappe.get_doc(
 			doctype="Server Script",
 			name="rate_limited_server_script",
diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py
index 640048d93a..6a9aeba075 100644
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@@ -1018,7 +1018,7 @@ def sign_up(email: str, full_name: str, redirect_to: str) -> tuple[int, str]:
 
 
 @frappe.whitelist(allow_guest=True)
-@rate_limit(limit=get_password_reset_limit, seconds=24 * 60 * 60)
+@rate_limit(limit=get_password_reset_limit, seconds=60 * 60)
 def reset_password(user: str) -> str:
 	if user == "Administrator":
 		return "not allowed"
diff --git a/frappe/rate_limiter.py b/frappe/rate_limiter.py
index 332a5a8070..adeaf94a31 100644
--- a/frappe/rate_limiter.py
+++ b/frappe/rate_limiter.py
@@ -138,7 +138,7 @@ def rate_limit(
 			if not identity:
 				frappe.throw(_("Either key or IP flag is required."))
 
-			cache_key = f"rl:{frappe.form_dict.cmd}:{identity}"
+			cache_key = frappe.cache.make_key(f"rl:{frappe.form_dict.cmd}:{identity}")
 
 			value = frappe.cache.get(cache_key)
 			if not value:
diff --git a/frappe/utils/password.py b/frappe/utils/password.py
index 3ee92eabda..f5f83cef1e 100644
--- a/frappe/utils/password.py
+++ b/frappe/utils/password.py
@@ -215,4 +215,4 @@ def get_encryption_key():
 
 
 def get_password_reset_limit():
-	return frappe.db.get_single_value("System Settings", "password_reset_limit") or 0
+	return frappe.get_system_settings("password_reset_limit") or 3

From e7b3a785c1dfa0d3d187f6b33045a3522490bd5b Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Wed, 31 Jan 2024 16:12:56 +0530
Subject: [PATCH 33/64] fix: typeerror in onboarding_tours.js

---
 frappe/public/js/onboarding_tours/onboarding_tours.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/onboarding_tours/onboarding_tours.js b/frappe/public/js/onboarding_tours/onboarding_tours.js
index 6bd3a62012..a8ee97a586 100644
--- a/frappe/public/js/onboarding_tours/onboarding_tours.js
+++ b/frappe/public/js/onboarding_tours/onboarding_tours.js
@@ -265,7 +265,7 @@ frappe.ui.init_onboarding_tour = () => {
 	typeof frappe.boot.user.onboarding_status == "undefined" &&
 		frappe.boot.user.onboarding_status == {};
 	let route = frappe.router.current_route;
-	if (route[0] === "") return;
+	if (route?.[0] === "") return;
 
 	let tour_name;
 	let matching_tours = [];

From 4d2c4f020bac93bdd648c4d3c1f514891c41d908 Mon Sep 17 00:00:00 2001
From: marination <maricadsouza221197@gmail.com>
Date: Wed, 31 Jan 2024 12:54:20 +0100
Subject: [PATCH 34/64] fix: Check if header/footer html exists before trying
 to extract

- issue: AttributeError: 'NoneType' object has no attribute 'extract', when no letterhead html was present
---
 frappe/utils/pdf.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py
index be25b8cd48..3e6966ae0a 100644
--- a/frappe/utils/pdf.py
+++ b/frappe/utils/pdf.py
@@ -238,7 +238,8 @@ def prepare_header_footer(soup: BeautifulSoup):
 
 	# extract header and footer
 	for html_id in ("header-html", "footer-html"):
-		if content := soup.find(id=html_id).extract():
+		if content := soup.find(id=html_id):
+			content = content.extract()
 			# `header/footer-html` are extracted, rendered as html
 			# and passed in wkhtmltopdf options (as '--header/footer-html')
 			# Remove instances of them from main content for render_template

From 09919a3e581e9bb3d8cd1bfde0b2081902a95ea1 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Wed, 31 Jan 2024 12:55:41 +0100
Subject: [PATCH 35/64] fix: use the correct doc method

---
 frappe/handler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/handler.py b/frappe/handler.py
index d889c67b23..d875e9bd57 100644
--- a/frappe/handler.py
+++ b/frappe/handler.py
@@ -250,7 +250,7 @@ def check_write_permission(doctype: str = None, name: str = None):
 	if doctype and name:
 		try:
 			doc = frappe.get_doc(doctype, name)
-			doc.has_permission("write")
+			doc.check_permission("write")
 		except frappe.DoesNotExistError:
 			# doc has not been inserted yet, name is set to "new-some-doctype"
 			check_doctype = True

From d2d669b72053de18e8638b54dca0d39a83ab4fa8 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Wed, 31 Jan 2024 18:09:20 +0530
Subject: [PATCH 36/64] fix: don't notify links if not public

---
 frappe/desk/doctype/note/note.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/desk/doctype/note/note.py b/frappe/desk/doctype/note/note.py
index 9cc14cbc1e..ef64d17f79 100644
--- a/frappe/desk/doctype/note/note.py
+++ b/frappe/desk/doctype/note/note.py
@@ -28,6 +28,9 @@ class Note(Document):
 			# expire this notification in a week (default)
 			self.expire_notification_on = frappe.utils.add_days(self.creation, 7)
 
+		if not self.public and self.notify_on_login:
+			self.notify_on_login = 0
+
 		if not self.content:
 			self.content = "<span></span>"
 

From ffb9c9cf6ab2911eb00176482d94d99ad96a3ac7 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Wed, 31 Jan 2024 18:24:15 +0530
Subject: [PATCH 37/64] fix: clear sitemap cache periodically

---
 frappe/website/router.py |  2 ++
 frappe/www/sitemap.py    | 61 ++++++++++++++++++++--------------------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/frappe/website/router.py b/frappe/website/router.py
index 3bffa78fee..332be38cda 100644
--- a/frappe/website/router.py
+++ b/frappe/website/router.py
@@ -322,7 +322,9 @@ def clear_routing_cache():
 	from frappe.website.doctype.web_form.web_form import get_published_web_forms
 	from frappe.website.doctype.web_page.web_page import get_dynamic_web_pages
 	from frappe.website.page_renderers.document_page import _find_matching_document_webview
+	from frappe.www.sitemap import get_public_pages_from_doctypes
 
 	_find_matching_document_webview.clear_cache()
 	get_dynamic_web_pages.clear_cache()
 	get_published_web_forms.clear_cache()
+	get_public_pages_from_doctypes.clear_cache()
diff --git a/frappe/www/sitemap.py b/frappe/www/sitemap.py
index ebe6846a39..1e8dea4038 100644
--- a/frappe/www/sitemap.py
+++ b/frappe/www/sitemap.py
@@ -6,6 +6,7 @@ from urllib.parse import quote
 import frappe
 from frappe.model.document import get_controller
 from frappe.utils import get_url, nowdate
+from frappe.utils.caching import redis_cache
 from frappe.website.router import get_pages
 
 no_cache = 1
@@ -31,42 +32,40 @@ def get_context(context):
 	return {"links": links}
 
 
+@redis_cache(ttl=6 * 60 * 60)
 def get_public_pages_from_doctypes():
 	"""Return pages from doctypes that are publicly accessible."""
 
-	def get_sitemap_routes():
-		routes = {}
-		doctypes_with_web_view = frappe.get_all(
-			"DocType",
-			filters={"has_web_view": True, "allow_guest_to_view": True},
-			pluck="name",
-		)
+	routes = {}
+	doctypes_with_web_view = frappe.get_all(
+		"DocType",
+		filters={"has_web_view": True, "allow_guest_to_view": True},
+		pluck="name",
+	)
 
-		for doctype in doctypes_with_web_view:
-			controller = get_controller(doctype)
-			meta = frappe.get_meta(doctype)
-			condition_field = meta.is_published_field or controller.website.condition_field
+	for doctype in doctypes_with_web_view:
+		controller = get_controller(doctype)
+		meta = frappe.get_meta(doctype)
+		condition_field = meta.is_published_field or controller.website.condition_field
 
-			if not condition_field:
-				continue
+		if not condition_field:
+			continue
 
-			try:
-				res = frappe.get_all(
-					doctype,
-					fields=["route", "name", "modified"],
-					filters={condition_field: True},
-				)
-			except Exception as e:
-				if not frappe.db.is_missing_column(e):
-					raise e
+		try:
+			res = frappe.get_all(
+				doctype,
+				fields=["route", "name", "modified"],
+				filters={condition_field: True},
+			)
+		except Exception as e:
+			if not frappe.db.is_missing_column(e):
+				raise e
 
-			for r in res:
-				routes[r.route] = {
-					"doctype": doctype,
-					"name": r.name,
-					"modified": r.modified,
-				}
+		for r in res:
+			routes[r.route] = {
+				"doctype": doctype,
+				"name": r.name,
+				"modified": r.modified,
+			}
 
-		return routes
-
-	return frappe.cache.get_value("sitemap_routes", get_sitemap_routes)
+	return routes

From ca293c9aa328183eaf9940147c80fd41c6bce636 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Wed, 31 Jan 2024 18:37:12 +0530
Subject: [PATCH 38/64] fix: Enqueue deletion of dynamic link after comitting

---
 frappe/model/delete_doc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py
index b6ab2546bf..6a92971c72 100644
--- a/frappe/model/delete_doc.py
+++ b/frappe/model/delete_doc.py
@@ -133,6 +133,7 @@ def delete_doc(
 					doctype=doc.doctype,
 					name=doc.name,
 					now=frappe.flags.in_test,
+					enqueue_after_commit=True,
 				)
 
 		# clear cache for Document

From 0458544c3b7c69dd9affcb402d9c68d094f78979 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Wed, 31 Jan 2024 21:23:04 +0530
Subject: [PATCH 39/64] refactor!: Simplify asset caching (#24649)

Few changes:

1. Client side asset caching is needlessly complicated with 2 layer of
caching. Browser already caches things, moving them again to
localstorage is uselss and counterproductive because of limits on localstorage.
2. `frappe.assets` was an object that should've been a class.
3. lots of old js syntax.
---
 frappe/public/js/frappe/assets.js | 164 +++++++++---------------------
 1 file changed, 48 insertions(+), 116 deletions(-)

diff --git a/frappe/public/js/frappe/assets.js b/frappe/public/js/frappe/assets.js
index 98b025ebea..53883289a1 100644
--- a/frappe/public/js/frappe/assets.js
+++ b/frappe/public/js/frappe/assets.js
@@ -19,11 +19,22 @@ frappe.require = function (items, callback) {
 	});
 };
 
-frappe.assets = {
-	check: function () {
+class AssetManager {
+	constructor() {
+		this._executed = [];
+		this._handlers = {
+			js: function (txt) {
+				frappe.dom.eval(txt);
+			},
+			css: function (txt) {
+				frappe.dom.set_style(txt);
+			},
+		};
+	}
+	check() {
 		// if version is different then clear localstorage
 		if (window._version_number != localStorage.getItem("_version_number")) {
-			frappe.assets.clear_local_storage();
+			this.clear_local_storage();
 			console.log("Cleared App Cache.");
 		}
 
@@ -33,160 +44,79 @@ frappe.assets = {
 			// Evict cache if page is reloaded within 10 seconds. Which could be user trying to
 			// refresh if things feel broken.
 			if ((not_updated_since < 5000 && is_reload()) || not_updated_since > 2 * 86400000) {
-				frappe.assets.clear_local_storage();
+				this.clear_local_storage();
 			}
 		} else {
-			frappe.assets.clear_local_storage();
+			this.clear_local_storage();
 		}
 
-		frappe.assets.init_local_storage();
-	},
+		this.init_local_storage();
+	}
 
-	init_local_storage: function () {
+	init_local_storage() {
 		localStorage._last_load = new Date();
 		localStorage._version_number = window._version_number;
 		if (frappe.boot) localStorage.metadata_version = frappe.boot.metadata_version;
-	},
+	}
 
-	clear_local_storage: function () {
-		$.each(
-			["_last_load", "_version_number", "metadata_version", "page_info", "last_visited"],
-			function (i, key) {
-				localStorage.removeItem(key);
-			}
+	clear_local_storage() {
+		["_last_load", "_version_number", "metadata_version", "page_info", "last_visited"].forEach(
+			(key) => localStorage.removeItem(key)
 		);
 
 		// clear assets
-		for (var key in localStorage) {
+		for (let key in localStorage) {
 			if (
-				key.indexOf("desk_assets:") === 0 ||
-				key.indexOf("_page:") === 0 ||
-				key.indexOf("_doctype:") === 0 ||
-				key.indexOf("preferred_breadcrumbs:") === 0
+				key.startsWith("_page:") ||
+				key.startsWith("_doctype:") ||
+				key.startsWith("preferred_breadcrumbs:")
 			) {
 				localStorage.removeItem(key);
 			}
 		}
 		console.log("localStorage cleared");
-	},
+	}
 
-	// keep track of executed assets
-	executed_: [],
+	eval_assets(path, content) {
+		if (!this._executed.includes(path)) {
+			this._handlers[this.extn(path)](content);
+			this._executed.push(path);
+		}
+	}
 
-	// pass on to the handler to set
-	execute: function (items, callback) {
-		var to_fetch = [];
-		for (var i = 0, l = items.length; i < l; i++) {
-			if (!frappe.assets.exists(items[i])) {
-				to_fetch.push(items[i]);
-			}
-		}
-		if (to_fetch.length) {
-			frappe.assets.fetch(to_fetch, function () {
-				frappe.assets.eval_assets(items, callback);
-			});
-		} else {
-			frappe.assets.eval_assets(items, callback);
-		}
-	},
-
-	eval_assets: function (items, callback) {
-		for (var i = 0, l = items.length; i < l; i++) {
-			// execute js/css if not already.
-			var path = items[i];
-			if (frappe.assets.executed_.indexOf(path) === -1) {
-				// execute
-				frappe.assets.handler[frappe.assets.extn(path)](frappe.assets.get(path), path);
-				frappe.assets.executed_.push(path);
-			}
-		}
-		callback && callback();
-	},
-
-	// check if the asset exists in
-	// localstorage
-	exists: function (src) {
-		if (frappe.assets.executed_.indexOf(src) !== -1) {
-			return true;
-		}
-		if (frappe.boot.developer_mode) {
-			return false;
-		}
-		if (frappe.assets.get(src)) {
-			return true;
-		} else {
-			return false;
-		}
-	},
-
-	// load an asset via
-	fetch: function (items, callback) {
+	execute(items, callback) {
 		// this is virtual page load, only get the the source
-		// *without* the template
-
-		if (items.length === 0) {
-			callback();
-			return;
-		}
+		let me = this;
 
 		const version_string =
 			frappe.boot.developer_mode || window.dev_server ? Date.now() : window._version_number;
 
-		async function fetch_item(item) {
+		async function fetch_item(path) {
 			// Add the version to the URL to bust the cache for non-bundled assets
-			let url = new URL(item, window.location.origin);
+			let url = new URL(path, window.location.origin);
 
-			if (!item.includes(".bundle.") && !url.searchParams.get("v")) {
+			if (!path.includes(".bundle.") && !url.searchParams.get("v")) {
 				url.searchParams.append("v", version_string);
 			}
 			const response = await fetch(url.toString());
 			const body = await response.text();
-			frappe.assets.add(item, body);
+			me.eval_assets(path, body);
 		}
 
 		frappe.dom.freeze();
 		const fetch_promises = items.map(fetch_item);
 		Promise.all(fetch_promises).then(() => {
 			frappe.dom.unfreeze();
-			callback();
+			callback?.();
 		});
-	},
+	}
 
-	add: function (src, txt) {
-		if ("localStorage" in window) {
-			try {
-				frappe.assets.set(src, txt);
-			} catch (e) {
-				// if quota is exceeded, clear local storage and set item
-				frappe.assets.clear_local_storage();
-				frappe.assets.set(src, txt);
-			}
-		}
-	},
-
-	get: function (src) {
-		return localStorage.getItem("desk_assets:" + src);
-	},
-
-	set: function (src, txt) {
-		localStorage.setItem("desk_assets:" + src, txt);
-	},
-
-	extn: function (src) {
+	extn(src) {
 		if (src.indexOf("?") != -1) {
 			src = src.split("?").slice(-1)[0];
 		}
 		return src.split(".").slice(-1)[0];
-	},
-
-	handler: {
-		js: function (txt, src) {
-			frappe.dom.eval(txt);
-		},
-		css: function (txt, src) {
-			frappe.dom.set_style(txt);
-		},
-	},
+	}
 
 	bundled_asset(path, is_rtl = null) {
 		if (!path.startsWith("/assets") && path.includes(".bundle.")) {
@@ -197,8 +127,8 @@ frappe.assets = {
 			return path;
 		}
 		return path;
-	},
-};
+	}
+}
 
 function is_reload() {
 	try {
@@ -211,3 +141,5 @@ function is_reload() {
 		return true;
 	}
 }
+
+frappe.assets = new AssetManager();

From 3634d11b6ad58aef2270f87abbafc9cd6395bb00 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 1 Feb 2024 10:44:00 +0530
Subject: [PATCH 40/64] fix: Retry contact update if it fails due to conflict
 (#24654)

---
 frappe/core/doctype/user/user.py | 53 +++++++++++++++++---------------
 1 file changed, 28 insertions(+), 25 deletions(-)

diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py
index 6a9aeba075..aacad57e9a 100644
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@@ -1254,34 +1254,37 @@ def create_contact(user, ignore_links=False, ignore_mandatory=False):
 		except frappe.DuplicateEntryError:
 			pass
 	else:
-		contact = frappe.get_doc("Contact", contact_name)
-		contact.first_name = user.first_name
-		contact.last_name = user.last_name
-		contact.gender = user.gender
+		try:
+			contact = frappe.get_doc("Contact", contact_name)
+			contact.first_name = user.first_name
+			contact.last_name = user.last_name
+			contact.gender = user.gender
 
-		# Add mobile number if phone does not exists in contact
-		if user.phone and not any(new_contact.phone == user.phone for new_contact in contact.phone_nos):
-			# Set primary phone if there is no primary phone number
-			contact.add_phone(
-				user.phone,
-				is_primary_phone=not any(
-					new_contact.is_primary_phone == 1 for new_contact in contact.phone_nos
-				),
-			)
+			# Add mobile number if phone does not exists in contact
+			if user.phone and not any(new_contact.phone == user.phone for new_contact in contact.phone_nos):
+				# Set primary phone if there is no primary phone number
+				contact.add_phone(
+					user.phone,
+					is_primary_phone=not any(
+						new_contact.is_primary_phone == 1 for new_contact in contact.phone_nos
+					),
+				)
 
-		# Add mobile number if mobile does not exists in contact
-		if user.mobile_no and not any(
-			new_contact.phone == user.mobile_no for new_contact in contact.phone_nos
-		):
-			# Set primary mobile if there is no primary mobile number
-			contact.add_phone(
-				user.mobile_no,
-				is_primary_mobile_no=not any(
-					new_contact.is_primary_mobile_no == 1 for new_contact in contact.phone_nos
-				),
-			)
+			# Add mobile number if mobile does not exists in contact
+			if user.mobile_no and not any(
+				new_contact.phone == user.mobile_no for new_contact in contact.phone_nos
+			):
+				# Set primary mobile if there is no primary mobile number
+				contact.add_phone(
+					user.mobile_no,
+					is_primary_mobile_no=not any(
+						new_contact.is_primary_mobile_no == 1 for new_contact in contact.phone_nos
+					),
+				)
 
-		contact.save(ignore_permissions=True)
+			contact.save(ignore_permissions=True)
+		except frappe.TimestampMismatchError:
+			raise frappe.RetryBackgroundJobError
 
 
 def get_restricted_ip_list(user):

From 78665d98b2d4a4cff3577854741d69d244c4b6f9 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 1 Feb 2024 10:57:38 +0530
Subject: [PATCH 41/64] fix: Bail if no SMTP acccount can be found

Currently we give this weird error instead of just saying no outgoing email account is configured.

```
AttributeError: 'NoneType' object has no attribute 'get_smtp_server'
  File "frappe/email/queue.py", line 154, in flush
    email_queue.send()
  File "frappe/email/doctype/email_queue/email_queue.py", line 160, in send
    with SendMailContext(self, smtp_server_instance) as ctx:
  File "frappe/email/doctype/email_queue/email_queue.py", line 236, in __init__
    self.smtp_server: SMTPServer = smtp_server_instance or self.email_account_doc.get_smtp_server()
```
---
 frappe/email/doctype/email_queue/email_queue.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py
index 91ca2bbdbb..f5da857576 100644
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@@ -131,12 +131,12 @@ class EmailQueue(Document):
 	def attachments_list(self):
 		return json.loads(self.attachments) if self.attachments else []
 
-	def get_email_account(self):
+	def get_email_account(self, raise_error=False):
 		if self.email_account:
 			return frappe.get_cached_doc("Email Account", self.email_account)
 
 		return EmailAccount.find_outgoing(
-			match_by_email=self.sender, match_by_doctype=self.reference_doctype
+			match_by_email=self.sender, match_by_doctype=self.reference_doctype, _raise_error=raise_error
 		)
 
 	def is_to_be_sent(self):
@@ -233,7 +233,7 @@ class SendMailContext:
 		smtp_server_instance: SMTPServer = None,
 	):
 		self.queue_doc: EmailQueue = queue_doc
-		self.email_account_doc = queue_doc.get_email_account()
+		self.email_account_doc = queue_doc.get_email_account(raise_error=True)
 
 		self.smtp_server: SMTPServer = smtp_server_instance or self.email_account_doc.get_smtp_server()
 
@@ -733,7 +733,7 @@ class QueueBuilder:
 			recipients = list(set([r] + self.final_cc() + self.bcc))
 			q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True)
 			if not smtp_server_instance:
-				email_account = q.get_email_account()
+				email_account = q.get_email_account(raise_error=True)
 				smtp_server_instance = email_account.get_smtp_server()
 
 			with suppress(Exception):

From dae99eb53c4606267c847ba7e301395da354004b Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 1 Feb 2024 11:07:27 +0530
Subject: [PATCH 42/64] fix: Fetch SMTP server inside context

Currently if smtp server creation fails email queue is endlessly
retried. It should just fail if there's no outgoing account.
---
 frappe/email/doctype/email_queue/email_queue.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py
index f5da857576..aff8782abf 100644
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@@ -158,6 +158,7 @@ class EmailQueue(Document):
 			return
 
 		with SendMailContext(self, smtp_server_instance) as ctx:
+			ctx.fetch_smtp_server()
 			message = None
 			for recipient in self.recipients:
 				if recipient.is_mail_sent():
@@ -233,14 +234,16 @@ class SendMailContext:
 		smtp_server_instance: SMTPServer = None,
 	):
 		self.queue_doc: EmailQueue = queue_doc
-		self.email_account_doc = queue_doc.get_email_account(raise_error=True)
-
-		self.smtp_server: SMTPServer = smtp_server_instance or self.email_account_doc.get_smtp_server()
-
+		self.smtp_server: SMTPServer = smtp_server_instance
 		self.sent_to_atleast_one_recipient = any(
 			rec.recipient for rec in self.queue_doc.recipients if rec.is_mail_sent()
 		)
 
+	def fetch_smtp_server(self):
+		self.email_account_doc = self.queue_doc.get_email_account(raise_error=True)
+		if not self.smtp_server:
+			self.smtp_server = self.email_account_doc.get_smtp_server()
+
 	def __enter__(self):
 		self.queue_doc.update_status(status="Sending", commit=True)
 		return self

From 12a246d697304ea343fb0c88c12f0dbb5a730a65 Mon Sep 17 00:00:00 2001
From: jll-02 <63648645+jll-02@users.noreply.github.com>
Date: Thu, 1 Feb 2024 08:15:00 +0100
Subject: [PATCH 43/64] chore(deps): removes unused cariocffi dependency

---
 pyproject.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 5553a52e64..6b27ea77b9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,7 +31,6 @@ dependencies = [
     "beautifulsoup4~=4.12.2",
     "bleach-allowlist~=1.0.3",
     "bleach[css]~=6.0.0",
-    "cairocffi==1.5.1",
     "chardet~=5.1.0",
     "croniter~=2.0.1",
     "cryptography~=41.0.3",

From 281c1de6391b10cc15f319af8e5551cba2d1de63 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Thu, 1 Feb 2024 10:51:57 +0100
Subject: [PATCH 44/64] fix(Data Export): translate multiselect

---
 frappe/core/doctype/data_export/data_export.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/data_export/data_export.js b/frappe/core/doctype/data_export/data_export.js
index a819ed7fe3..1808578d31 100644
--- a/frappe/core/doctype/data_export/data_export.js
+++ b/frappe/core/doctype/data_export/data_export.js
@@ -153,7 +153,7 @@ const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {
 
 	const options = fields.map((df) => {
 		return {
-			label: df.label,
+			label: __(df.label),
 			value: df.fieldname,
 			danger: df.reqd,
 			checked: 1,
@@ -163,7 +163,7 @@ const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {
 	const multicheck_control = frappe.ui.form.make_control({
 		parent: parent_wrapper,
 		df: {
-			label: doctype,
+			label: __(doctype),
 			fieldname: doctype + "_fields",
 			fieldtype: "MultiCheck",
 			options: options,

From 833d108614ad2b6a89fe563a04817378ac6ba4eb Mon Sep 17 00:00:00 2001
From: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
Date: Thu, 1 Feb 2024 11:35:25 +0100
Subject: [PATCH 45/64] fix: remove _lt from frappe.model.std_fields (#24662)

---
 frappe/model/__init__.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py
index d3d120c872..18d7244179 100644
--- a/frappe/model/__init__.py
+++ b/frappe/model/__init__.py
@@ -134,22 +134,22 @@ log_types = (
 )
 
 std_fields = [
-	{"fieldname": "name", "fieldtype": "Link", "label": _lt("ID")},
-	{"fieldname": "owner", "fieldtype": "Link", "label": _lt("Created By"), "options": "User"},
-	{"fieldname": "idx", "fieldtype": "Int", "label": _lt("Index")},
-	{"fieldname": "creation", "fieldtype": "Datetime", "label": _lt("Created On")},
-	{"fieldname": "modified", "fieldtype": "Datetime", "label": _lt("Last Updated On")},
+	{"fieldname": "name", "fieldtype": "Link", "label": "ID"},
+	{"fieldname": "owner", "fieldtype": "Link", "label": "Created By", "options": "User"},
+	{"fieldname": "idx", "fieldtype": "Int", "label": "Index"},
+	{"fieldname": "creation", "fieldtype": "Datetime", "label": "Created On"},
+	{"fieldname": "modified", "fieldtype": "Datetime", "label": "Last Updated On"},
 	{
 		"fieldname": "modified_by",
 		"fieldtype": "Link",
-		"label": _lt("Last Updated By"),
+		"label": "Last Updated By",
 		"options": "User",
 	},
-	{"fieldname": "_user_tags", "fieldtype": "Data", "label": _lt("Tags")},
-	{"fieldname": "_liked_by", "fieldtype": "Data", "label": _lt("Liked By")},
-	{"fieldname": "_comments", "fieldtype": "Text", "label": _lt("Comments")},
-	{"fieldname": "_assign", "fieldtype": "Text", "label": _lt("Assigned To")},
-	{"fieldname": "docstatus", "fieldtype": "Int", "label": _lt("Document Status")},
+	{"fieldname": "_user_tags", "fieldtype": "Data", "label": "Tags"},
+	{"fieldname": "_liked_by", "fieldtype": "Data", "label": "Liked By"},
+	{"fieldname": "_comments", "fieldtype": "Text", "label": "Comments"},
+	{"fieldname": "_assign", "fieldtype": "Text", "label": "Assigned To"},
+	{"fieldname": "docstatus", "fieldtype": "Int", "label": "Document Status"},
 ]
 
 

From 4553de8ba0694105aee7fec37f1056a79eeea6e3 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 1 Feb 2024 16:22:07 +0530
Subject: [PATCH 46/64] fix: never show virtual fields in list view (#24666)

* fix: never render virtual fields in list view

* fix: Hide "in list view" for virtual columns
---
 frappe/core/doctype/docfield/docfield.json                    | 3 ++-
 frappe/core/doctype/docfield/docfield.py                      | 1 +
 .../doctype/customize_form_field/customize_form_field.json    | 3 ++-
 .../doctype/customize_form_field/customize_form_field.py      | 1 +
 frappe/public/js/frappe/list/base_list.js                     | 4 +++-
 frappe/public/js/frappe/list/list_view.js                     | 2 +-
 6 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/frappe/core/doctype/docfield/docfield.json b/frappe/core/doctype/docfield/docfield.json
index 22b84d1cbc..7b188569d7 100644
--- a/frappe/core/doctype/docfield/docfield.json
+++ b/frappe/core/doctype/docfield/docfield.json
@@ -157,6 +157,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval:!doc.is_virtual",
    "fieldname": "in_list_view",
    "fieldtype": "Check",
    "label": "In List View",
@@ -580,7 +581,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-11-16 11:26:56.364594",
+ "modified": "2024-02-01 15:55:44.007917",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocField",
diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py
index 01fa56a9ce..ed663c3bba 100644
--- a/frappe/core/doctype/docfield/docfield.py
+++ b/frappe/core/doctype/docfield/docfield.py
@@ -117,6 +117,7 @@ class DocField(Document):
 		unique: DF.Check
 		width: DF.Data | None
 	# end: auto-generated types
+
 	def get_link_doctype(self):
 		"""Return the Link doctype for the `docfield` (if applicable).
 
diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.json b/frappe/custom/doctype/customize_form_field/customize_form_field.json
index 67c6c8ba95..0dfd7a6c45 100644
--- a/frappe/custom/doctype/customize_form_field/customize_form_field.json
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.json
@@ -130,6 +130,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval:!doc.is_virtual",
    "fieldname": "in_list_view",
    "fieldtype": "Check",
    "label": "In List View"
@@ -483,7 +484,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-12-08 15:52:37.525003",
+ "modified": "2024-02-01 15:56:39.171633",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",
diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.py b/frappe/custom/doctype/customize_form_field/customize_form_field.py
index 59b0155a98..76ab6535e3 100644
--- a/frappe/custom/doctype/customize_form_field/customize_form_field.py
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.py
@@ -110,4 +110,5 @@ class CustomizeFormField(Document):
 		unique: DF.Check
 		width: DF.Data | None
 	# end: auto-generated types
+
 	pass
diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js
index da76488b9c..a96ebb9157 100644
--- a/frappe/public/js/frappe/list/base_list.js
+++ b/frappe/public/js/frappe/list/base_list.js
@@ -132,7 +132,9 @@ frappe.views.BaseList = class BaseList {
 			frappe.meta.has_field(doctype, fieldname) ||
 			fieldname === "_seen";
 
-		if (!is_valid_field) {
+		let is_virtual = this.meta.fields.find((df) => df.fieldname == fieldname)?.is_virtual;
+
+		if (!is_valid_field || is_virtual) {
 			return;
 		}
 
diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js
index eb82f3338f..72c9ed8a44 100644
--- a/frappe/public/js/frappe/list/list_view.js
+++ b/frappe/public/js/frappe/list/list_view.js
@@ -374,7 +374,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 					if (frappe.has_indicator(this.doctype) && df.fieldname === "status") {
 						return false;
 					}
-					if (!df.in_list_view) {
+					if (!df.in_list_view || df.is_virtual) {
 						return false;
 					}
 					return df.fieldname !== this.meta.title_field;

From aed358609508182554294cc06029b6ae739e02ba Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Thu, 1 Feb 2024 17:10:21 +0530
Subject: [PATCH 47/64] fix: enable syntax highlighting for `PythonExpression`
 code fields (#24669)

---
 frappe/public/js/frappe/form/controls/code.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js
index 05e8fde29c..4efd92bd8d 100644
--- a/frappe/public/js/frappe/form/controls/code.js
+++ b/frappe/public/js/frappe/form/controls/code.js
@@ -153,6 +153,7 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex
 			JS: "ace/mode/javascript",
 			Python: "ace/mode/python",
 			Py: "ace/mode/python",
+			PythonExpression: "ace/mode/python",
 			HTML: "ace/mode/html",
 			CSS: "ace/mode/css",
 			Markdown: "ace/mode/markdown",

From 4d63a12ab6cbc5b506c5c53350c65228d4f12e0a Mon Sep 17 00:00:00 2001
From: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
Date: Thu, 1 Feb 2024 12:41:41 +0100
Subject: [PATCH 48/64] fix(Data Import): scheduler not needed in dev mode
 (#24667)

---
 frappe/core/doctype/data_import/data_import.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py
index c674122f30..b63b5cd2a7 100644
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@@ -92,7 +92,8 @@ class DataImport(Document):
 	def start_import(self):
 		from frappe.utils.scheduler import is_scheduler_inactive
 
-		if is_scheduler_inactive() and not frappe.flags.in_test:
+		run_now = frappe.flags.in_test or frappe.conf.developer_mode
+		if is_scheduler_inactive() and not run_now:
 			frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive"))
 
 		job_id = f"data_import::{self.name}"
@@ -105,7 +106,7 @@ class DataImport(Document):
 				event="data_import",
 				job_id=job_id,
 				data_import=self.name,
-				now=frappe.conf.developer_mode or frappe.flags.in_test,
+				now=run_now,
 			)
 			return True
 

From aad0c19f3a01c0cc41b383c65c71c92c57abc8b6 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 1 Feb 2024 17:57:25 +0530
Subject: [PATCH 49/64] fix: lazy translate circular imports (#24672)

* Revert "fix: remove _lt from frappe.model.std_fields (#24662)"

This reverts commit 833d108614ad2b6a89fe563a04817378ac6ba4eb.

* fix: circular import
---
 frappe/__init__.py                 | 40 ++++++++++++++++++++++++++++--
 frappe/gettext/extractors/utils.py |  3 ++-
 frappe/translate.py                | 38 ----------------------------
 3 files changed, 40 insertions(+), 41 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 289e5c6e3d..ce7ce7699b 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -130,9 +130,45 @@ def _lt(msg: str, lang: str | None = None, context: str | None = None):
 
 	Note: Result is not guaranteed to equivalent to pure strings for all operations.
 	"""
-	from frappe.translate import LazyTranslate
+	return _LazyTranslate(msg, lang, context)
 
-	return LazyTranslate(msg, lang, context)
+
+@functools.total_ordering
+class _LazyTranslate:
+	__slots__ = ("msg", "lang", "context")
+
+	def __init__(self, msg: str, lang: str | None = None, context: str | None = None) -> None:
+		self.msg = msg
+		self.lang = lang
+		self.context = context
+
+	@property
+	def value(self) -> str:
+		return _(str(self.msg), self.lang, self.context)
+
+	def __str__(self):
+		return self.value
+
+	def __add__(self, other):
+		if isinstance(other, (str, _LazyTranslate)):
+			return self.value + str(other)
+		raise NotImplementedError
+
+	def __radd__(self, other):
+		if isinstance(other, (str, _LazyTranslate)):
+			return str(other) + self.value
+		return NotImplementedError
+
+	def __repr__(self) -> str:
+		return f"'{self.value}'"
+
+	# NOTE: it's required to override these methods and raise error as default behaviour will
+	# return `False` in all cases.
+	def __eq__(self, other):
+		raise NotImplementedError
+
+	def __lt__(self, other):
+		raise NotImplementedError
 
 
 def as_unicode(text, encoding: str = "utf-8") -> str:
diff --git a/frappe/gettext/extractors/utils.py b/frappe/gettext/extractors/utils.py
index 4becbbe64e..e088a8409b 100644
--- a/frappe/gettext/extractors/utils.py
+++ b/frappe/gettext/extractors/utils.py
@@ -1,7 +1,6 @@
 import re
 
 import frappe
-from frappe.model.utils import InvalidIncludePath, render_include
 
 TRANSLATE_PATTERN = re.compile(
 	r"_\(\s*"  # starts with literal `_(`, ignore following whitespace/newlines
@@ -36,6 +35,8 @@ def extract_messages_from_code(code):
 	"""
 	from jinja2 import TemplateError
 
+	from frappe.model.utils import InvalidIncludePath, render_include
+
 	try:
 		code = frappe.as_unicode(render_include(code))
 
diff --git a/frappe/translate.py b/frappe/translate.py
index 7e270eee09..0f9098876f 100644
--- a/frappe/translate.py
+++ b/frappe/translate.py
@@ -1029,44 +1029,6 @@ def print_language(language: str):
 	frappe.local.jenv = _jenv
 
 
-@functools.total_ordering
-class LazyTranslate:
-	__slots__ = ("msg", "lang", "context")
-
-	def __init__(self, msg: str, lang: str | None = None, context: str | None = None) -> None:
-		self.msg = msg
-		self.lang = lang
-		self.context = context
-
-	@property
-	def value(self) -> str:
-		return frappe._(str(self.msg), self.lang, self.context)
-
-	def __str__(self):
-		return self.value
-
-	def __add__(self, other):
-		if isinstance(other, (str, LazyTranslate)):
-			return self.value + str(other)
-		raise NotImplementedError
-
-	def __radd__(self, other):
-		if isinstance(other, (str, LazyTranslate)):
-			return str(other) + self.value
-		return NotImplementedError
-
-	def __repr__(self) -> str:
-		return f"'{self.value}'"
-
-	# NOTE: it's required to override these methods and raise error as default behaviour will
-	# return `False` in all cases.
-	def __eq__(self, other):
-		raise NotImplementedError
-
-	def __lt__(self, other):
-		raise NotImplementedError
-
-
 # Backward compatibility
 get_full_dict = get_all_translations
 load_lang = get_translations_from_apps

From c479a038a84f55e956a716a52b0423303b8a1440 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 1 Feb 2024 19:42:54 +0530
Subject: [PATCH 50/64] fix: Avoid enqueueing during install (#24679)

This isn't strictly required but if bench isn't running this can break
installation.
---
 frappe/core/doctype/role_profile/role_profile.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/role_profile/role_profile.py b/frappe/core/doctype/role_profile/role_profile.py
index c026a4926a..d0fd42edfa 100644
--- a/frappe/core/doctype/role_profile/role_profile.py
+++ b/frappe/core/doctype/role_profile/role_profile.py
@@ -25,7 +25,11 @@ class RoleProfile(Document):
 		self.name = self.role_profile
 
 	def on_update(self):
-		self.queue_action("update_all_users", now=frappe.flags.in_test, enqueue_after_commit=True)
+		self.queue_action(
+			"update_all_users",
+			now=frappe.flags.in_test or frappe.flags.in_install,
+			enqueue_after_commit=True,
+		)
 
 	def update_all_users(self):
 		"""Changes in role_profile reflected across all its user"""

From c09f48f5d7421f71149321487aac14ee4a2d2085 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Fri, 2 Feb 2024 07:03:22 +0100
Subject: [PATCH 51/64] feat: comply trackers with utm and add utm_content for
 a/b testing (#24521)

* feat: comply trackers with utm and add utm_content for a/b testing

* fix: make all other than source optional

this is plausible.io best practice

* fix: medium is not specified to be lowercase

* fix: ensure proper parameter quoting
---
 frappe/public/js/frappe/utils/utils.js        | 24 ++++++++++++++-----
 frappe/utils/data.py                          | 21 +++++++++++-----
 .../doctype/web_page_view/web_page_view.json  |  8 ++++++-
 .../doctype/web_page_view/web_page_view.py    |  2 ++
 .../website_analytics/website_analytics.js    |  1 +
 frappe/www/website_script.js                  |  9 +++----
 6 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js
index 18f93ab187..15785d514f 100644
--- a/frappe/public/js/frappe/utils/utils.js
+++ b/frappe/public/js/frappe/utils/utils.js
@@ -1705,6 +1705,8 @@ Object.assign(frappe.utils, {
 					fieldname: "source",
 					label: __("Source"),
 					fieldtype: "Data",
+					reqd: 1,
+					description: "The referrer (e.g. google, newsletter)",
 					default: localStorage.getItem("tracker_url:source"),
 				},
 				{
@@ -1719,25 +1721,35 @@ Object.assign(frappe.utils, {
 					fieldname: "medium",
 					label: __("Medium"),
 					fieldtype: "Data",
+					description: "Marketing medium (e.g. cpc, banner, email)",
 					default: localStorage.getItem("tracker_url:medium"),
 				},
+				{
+					fieldname: "content",
+					label: __("Content"),
+					fieldtype: "Data",
+					description: "Use to differentiate ad variants (e.g. A/B testing)",
+					default: localStorage.getItem("tracker_url:content"),
+				},
 			],
 			function (data) {
 				let url = data.url;
 				localStorage.setItem("tracker_url:url", data.url);
 
-				if (data.source) {
-					url += "?source=" + data.source;
-					localStorage.setItem("tracker_url:source", data.source);
-				}
+				url += "?utm_source=" + encodeURIComponent(data.source);
+				localStorage.setItem("tracker_url:source", data.source);
 				if (data.campaign) {
-					url += "&campaign=" + data.campaign;
+					url += "&utm_campaign=" + encodeURIComponent(data.campaign);
 					localStorage.setItem("tracker_url:campaign", data.campaign);
 				}
 				if (data.medium) {
-					url += "&medium=" + data.medium.toLowerCase();
+					url += "&utm_medium=" + encodeURIComponent(data.medium);
 					localStorage.setItem("tracker_url:medium", data.medium);
 				}
+				if (data.medium) {
+					url += "&utm_content=" + encodeURIComponent(data.content);
+					localStorage.setItem("tracker_url:content", data.content);
+				}
 
 				frappe.utils.copy_to_clipboard(url);
 
diff --git a/frappe/utils/data.py b/frappe/utils/data.py
index c1026b7807..612b9f191a 100644
--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@@ -2486,18 +2486,27 @@ def is_site_link(link: str) -> bool:
 	return urlparse(link).netloc == urlparse(frappe.utils.get_url()).netloc
 
 
-def add_trackers_to_url(url: str, source: str, campaign: str, medium: str = "email") -> str:
+def add_trackers_to_url(
+	url: str,
+	source: str,
+	campaign: str | None = None,
+	medium: str | None = None,
+	content: str | None = None,
+) -> str:
 	url_parts = list(urlparse(url))
 	if url_parts[0] == "mailto":
 		return url
 
-	trackers = {
-		"source": source,
-		"medium": medium,
-	}
+	trackers = {"utm_source": source}
+
+	if medium:
+		trackers["utm_medium"] = medium
 
 	if campaign:
-		trackers["campaign"] = campaign
+		trackers["utm_campaign"] = campaign
+
+	if content:
+		trackers["utm_content"] = content
 
 	query = dict(parse_qsl(url_parts[4])) | trackers
 
diff --git a/frappe/website/doctype/web_page_view/web_page_view.json b/frappe/website/doctype/web_page_view/web_page_view.json
index 2e514ffaec..57718c4cef 100644
--- a/frappe/website/doctype/web_page_view/web_page_view.json
+++ b/frappe/website/doctype/web_page_view/web_page_view.json
@@ -82,6 +82,12 @@
    "fieldtype": "Data",
    "label": "Medium",
    "read_only": 1
+  },
+  {
+   "fieldname": "content",
+   "fieldtype": "Data",
+   "label": "Content",
+   "read_only": 1
   }
  ],
  "in_create": 1,
@@ -111,4 +117,4 @@
  "sort_order": "DESC",
  "states": [],
  "title_field": "path"
-}
\ No newline at end of file
+}
diff --git a/frappe/website/doctype/web_page_view/web_page_view.py b/frappe/website/doctype/web_page_view/web_page_view.py
index 7c18d1ff66..05d482e0fd 100644
--- a/frappe/website/doctype/web_page_view/web_page_view.py
+++ b/frappe/website/doctype/web_page_view/web_page_view.py
@@ -47,6 +47,7 @@ def make_view_log(
 	source=None,
 	campaign=None,
 	medium=None,
+	content=None,
 	visitor_id=None,
 ):
 	if not is_tracking_enabled():
@@ -85,6 +86,7 @@ def make_view_log(
 	view.source = source
 	view.campaign = campaign
 	view.medium = (medium or "").lower()
+	view.content = content
 	view.visitor_id = visitor_id
 
 	try:
diff --git a/frappe/website/report/website_analytics/website_analytics.js b/frappe/website/report/website_analytics/website_analytics.js
index ac1445eaed..f163f5da81 100644
--- a/frappe/website/report/website_analytics/website_analytics.js
+++ b/frappe/website/report/website_analytics/website_analytics.js
@@ -38,6 +38,7 @@ frappe.query_reports["Website Analytics"] = {
 				{ value: "source", label: __("Source") },
 				{ value: "campaign", label: __("Campaign") },
 				{ value: "medium", label: __("Medium") },
+				{ value: "content", label: __("Content") },
 			],
 			default: "path",
 		},
diff --git a/frappe/www/website_script.js b/frappe/www/website_script.js
index a95b21313e..89d6fcd5c5 100644
--- a/frappe/www/website_script.js
+++ b/frappe/www/website_script.js
@@ -32,12 +32,13 @@ ga('send', 'pageview');
 						browser: browser.name,
 						version: browser.version,
 						user_tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
-						source: query_params.source,
-						medium: query_params.medium,
-						campaign: query_params.campaign,
+						source: query_params.source || query_params.utm_source,
+						medium: query_params.medium || query_params.utm_medium,
+						campaign: query_params.campaign || query_params.utm_campaign,
+						content: query_params.content || query_params.utm_content,
 						visitor_id: result.visitorId
 					})
 			})
 		})
 	}
-{% endif %}
\ No newline at end of file
+{% endif %}

From d26c67df75a95ef43d329eadd48d7998ea656856 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 2 Feb 2024 13:37:46 +0530
Subject: [PATCH 52/64] fix: virtual fields are never writable (#24693)

---
 frappe/public/js/frappe/model/perm.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/model/perm.js b/frappe/public/js/frappe/model/perm.js
index 77b92b046e..3358d67e55 100644
--- a/frappe/public/js/frappe/model/perm.js
+++ b/frappe/public/js/frappe/model/perm.js
@@ -203,7 +203,7 @@ $.extend(frappe.perm, {
 
 		// permission
 		if (p) {
-			if (p.write && !df.disabled) {
+			if (p.write && !df.disabled && !df.is_virtual) {
 				status = "Write";
 			} else if (p.read) {
 				status = "Read";

From 380b33d49ee9eac3962399b6d6a4b20a6a195c8a Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 2 Feb 2024 15:08:16 +0530
Subject: [PATCH 53/64] fix: Dashboard view realtime error (#24698)

filter_area doesn't exist on DashboardView, so this inherited function
breaks down.
---
 frappe/public/js/frappe/list/list_view.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js
index 72c9ed8a44..8b37c37836 100644
--- a/frappe/public/js/frappe/list/list_view.js
+++ b/frappe/public/js/frappe/list/list_view.js
@@ -1520,7 +1520,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 	}
 
 	avoid_realtime_update() {
-		if (this.filter_area.is_being_edited()) {
+		if (this.filter_area?.is_being_edited()) {
 			return true;
 		}
 		// this is set when a bulk operation is called from a list view which might update the list view

From 11baf6e03c7d37325d02b454f63468a8f7d2a57b Mon Sep 17 00:00:00 2001
From: Corentin Flr <10946971+cogk@users.noreply.github.com>
Date: Fri, 2 Feb 2024 11:03:55 +0100
Subject: [PATCH 54/64] fix(login): Escape translated strings (#24431)

* Use `tojson` instead of `json` Jinja filter
---
 frappe/templates/includes/login/login.js | 62 ++++++++++++------------
 1 file changed, 32 insertions(+), 30 deletions(-)

diff --git a/frappe/templates/includes/login/login.js b/frappe/templates/includes/login/login.js
index 3cff2e4652..204df64dff 100644
--- a/frappe/templates/includes/login/login.js
+++ b/frappe/templates/includes/login/login.js
@@ -21,7 +21,8 @@ login.bind_events = function () {
 		args.usr = frappe.utils.xss_sanitise(($("#login_email").val() || "").trim());
 		args.pwd = $("#login_password").val();
 		if (!args.usr || !args.pwd) {
-			frappe.msgprint('{{ _("Both login and password required") }}');
+			{# striptags is used to remove newlines, e is used for escaping #}
+			frappe.msgprint("{{ _('Both login and password required') | striptags | e }}");
 			return false;
 		}
 		login.call(args, null, "/login");
@@ -36,7 +37,7 @@ login.bind_events = function () {
 		args.redirect_to = frappe.utils.sanitise_redirect(frappe.utils.get_url_arg("redirect-to"));
 		args.full_name = frappe.utils.xss_sanitise(($("#signup_fullname").val() || "").trim());
 		if (!args.email || !validate_email(args.email) || !args.full_name) {
-			login.set_status('{{ _("Valid email and name required") }}', 'red');
+			login.set_status({{ _("Valid email and name required") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@@ -49,7 +50,7 @@ login.bind_events = function () {
 		args.cmd = "frappe.core.doctype.user.user.reset_password";
 		args.user = ($("#forgot_email").val() || "").trim();
 		if (!args.user) {
-			login.set_status('{{ _("Valid Login id required.") }}', 'red');
+			login.set_status({{ _("Valid Login id required.") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@@ -62,14 +63,14 @@ login.bind_events = function () {
 		args.cmd = "frappe.www.login.send_login_link";
 		args.email = ($("#login_with_email_link_email").val() || "").trim();
 		if (!args.email) {
-			login.set_status('{{ _("Valid Login id required.") }}', 'red');
+			login.set_status({{ _("Valid Login id required.") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args).then(() => {
-			login.set_status('{{ _("Login link sent to your email") }}', 'blue');
+			login.set_status({{ _("Login link sent to your email") | tojson }}, 'blue');
 			$("#login_with_email_link_email").val("");
 		}).catch(() => {
-			login.set_status('{{ _("Send login link") }}', 'blue');
+			login.set_status({{ _("Send login link") | tojson }}, 'blue');
 		});
 
 		return false;
@@ -79,10 +80,10 @@ login.bind_events = function () {
 		var input = $($(this).attr("toggle"));
 		if (input.attr("type") == "password") {
 			input.attr("type", "text");
-			$(this).text('{{ _("Hide") }}')
+			$(this).text({{ _("Hide") | tojson }})
 		} else {
 			input.attr("type", "password");
-			$(this).text('{{ _("Show") }}')
+			$(this).text({{ _("Show") | tojson }})
 		}
 	});
 
@@ -93,7 +94,7 @@ login.bind_events = function () {
 		args.usr = ($("#login_email").val() || "").trim();
 		args.pwd = $("#login_password").val();
 		if (!args.usr || !args.pwd) {
-			login.set_status('{{ _("Both login and password required") }}', 'red');
+			login.set_status({{ _("Both login and password required") | tojson }}, 'red');
 			return false;
 		}
 		login.call(args);
@@ -168,7 +169,7 @@ login.signup = function () {
 
 // Login
 login.call = function (args, callback, url="/") {
-	login.set_status('{{ _("Verifying...") }}', 'blue');
+	login.set_status({{ _("Verifying...") | tojson }}, 'blue');
 
 	return frappe.call({
 		type: "POST",
@@ -227,13 +228,13 @@ login.login_handlers = (function () {
 	var login_handlers = {
 		200: function (data) {
 			if (data.message == 'Logged In') {
-				login.set_status('{{ _("Success") }}', 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 				document.body.innerHTML = `{% include "templates/includes/splash_screen.html" %}`;
 				window.location.href = frappe.utils.sanitise_redirect(frappe.utils.get_url_arg("redirect-to")) || data.home_page;
 			} else if (data.message == 'Password Reset') {
 				window.location.href = frappe.utils.sanitise_redirect(data.redirect_to);
 			} else if (data.message == "No App") {
-				login.set_status("{{ _('Success') }}", 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 				if (localStorage) {
 					var last_visited =
 						localStorage.getItem("last_visited")
@@ -252,13 +253,13 @@ login.login_handlers = (function () {
 				}
 			} else if (window.location.hash === '#forgot') {
 				if (data.message === 'not found') {
-					login.set_status('{{ _("Not a valid user") }}', 'red');
+					login.set_status({{ _("Not a valid user") | tojson }}, 'red');
 				} else if (data.message == 'not allowed') {
-					login.set_status('{{ _("Not Allowed") }}', 'red');
+					login.set_status({{ _("Not Allowed") | tojson }}, 'red');
 				} else if (data.message == 'disabled') {
-					login.set_status('{{ _("Not Allowed: Disabled User") }}', 'red');
+					login.set_status({{ _("Not Allowed: Disabled User") | tojson }}, 'red');
 				} else {
-					login.set_status('{{ _("Instructions Emailed") }}', 'green');
+					login.set_status({{ _("Instructions Emailed") | tojson }}, 'green');
 				}
 
 
@@ -266,7 +267,7 @@ login.login_handlers = (function () {
 				if (cint(data.message[0]) == 0) {
 					login.set_status(data.message[1], 'red');
 				} else {
-					login.set_status('{{ _("Success") }}', 'green');
+					login.set_status({{ _("Success") | tojson }}, 'green');
 					frappe.msgprint(data.message[1])
 				}
 				//login.set_status(__(data.message), 'green');
@@ -274,7 +275,7 @@ login.login_handlers = (function () {
 
 			//OTP verification
 			if (data.verification && data.message != 'Logged In') {
-				login.set_status('{{ _("Success") }}', 'green');
+				login.set_status({{ _("Success") | tojson }}, 'green');
 
 				document.cookie = "tmp_id=" + data.tmp_id;
 
@@ -287,10 +288,10 @@ login.login_handlers = (function () {
 				}
 			}
 		},
-		401: get_error_handler('{{ _("Invalid Login. Try again.") }}'),
-		417: get_error_handler('{{ _("Oops! Something went wrong.") }}'),
-		404: get_error_handler('{{ _("User does not exist.")}}'),
-		500: get_error_handler('{{ _("Something went wrong.") }}')
+		401: get_error_handler({{ _("Invalid Login. Try again.") | tojson }}),
+		417: get_error_handler({{ _("Oops! Something went wrong.") | tojson }}),
+		404: get_error_handler({{ _("User does not exist.") | tojson }}),
+		500: get_error_handler({{ _("Something went wrong.") | tojson }})
 	};
 
 	return login_handlers;
@@ -322,7 +323,8 @@ var verify_token = function (event) {
 		args.otp = $("#login_token").val();
 		args.tmp_id = frappe.get_cookie('tmp_id');
 		if (!args.otp) {
-			frappe.msgprint('{{ _("Login token required") }}');
+			{# striptags is used to remove newlines, e is used for escaping #}
+			frappe.msgprint("{{ _('Login token required') | striptags | e }}");
 			return false;
 		}
 		login.call(args);
@@ -336,11 +338,11 @@ var request_otp = function (r) {
 		`<div id="twofactor_div">
 			<form class="form-verify">
 				<div class="page-card-head">
-					<span class="indicator blue" data-text="Verification">{{ _("Verification") }}</span>
+					<span class="indicator blue" data-text="Verification">{{ _("Verification") | e }}</span>
 				</div>
 				<div id="otp_div"></div>
-				<input type="text" id="login_token" autocomplete="off" class="form-control" placeholder="{{ _("Verification Code") }}" required="">
-				<button class="btn btn-sm btn-primary btn-block mt-3" id="verify_token">{{ _("Verify") }}</button>
+				<input type="text" id="login_token" autocomplete="off" class="form-control" placeholder="{{ _("Verification Code") | e }}" required="">
+				<button class="btn btn-sm btn-primary btn-block mt-3" id="verify_token">{{ _("Verify") | e }}</button>
 			</form>
 		</div>`
 	);
@@ -354,11 +356,11 @@ var continue_otp_app = function (setup, qrcode) {
 	var qrcode_div = $('<div class="text-muted" style="padding-bottom: 15px;"></div>');
 
 	if (setup) {
-		direction = $('<div>').attr('id', 'qr_info').html('{{ _("Enter Code displayed in OTP App.") }}');
+		direction = $('<div>').attr('id', 'qr_info').text({{ _("Enter Code displayed in OTP App.") | tojson }});
 		qrcode_div.append(direction);
 		$('#otp_div').prepend(qrcode_div);
 	} else {
-		direction = $('<div>').attr('id', 'qr_info').html('{{ _("OTP setup using OTP App was not completed. Please contact Administrator.") }}');
+		direction = $('<div>').attr('id', 'qr_info').text({{ _("OTP setup using OTP App was not completed. Please contact Administrator.") | tojson }});
 		qrcode_div.append(direction);
 		$('#otp_div').prepend(qrcode_div);
 	}
@@ -372,7 +374,7 @@ var continue_sms = function (setup, prompt) {
 		sms_div.append(prompt)
 		$('#otp_div').prepend(sms_div);
 	} else {
-		direction = $('<div>').attr('id', 'qr_info').html(prompt || '{{ _("SMS was not sent. Please contact Administrator.") }}');
+		direction = $('<div>').attr('id', 'qr_info').html(prompt || {{ _("SMS was not sent. Please contact Administrator.") | tojson }});
 		sms_div.append(direction);
 		$('#otp_div').prepend(sms_div)
 	}
@@ -386,7 +388,7 @@ var continue_email = function (setup, prompt) {
 		email_div.append(prompt)
 		$('#otp_div').prepend(email_div);
 	} else {
-		var direction = $('<div>').attr('id', 'qr_info').html(prompt || '{{ _("Verification code email not sent. Please contact Administrator.") }}');
+		var direction = $('<div>').attr('id', 'qr_info').html(prompt || {{ _("Verification code email not sent. Please contact Administrator.") | tojson }});
 		email_div.append(direction);
 		$('#otp_div').prepend(email_div);
 	}

From dd839e4d500c42f860cc8db46fb1a9ee5445f24d Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 2 Feb 2024 17:33:58 +0530
Subject: [PATCH 55/64] fix: lower default retention periods (#24697)

These can still be configured to different number. We are just changing
defaults. Why?

- Scheduled job log - errors get logged in error log anyway. Success log
  isn't that useful of 90 days?
- Prepared report - most reports are expired in a day or max a week.
  Rarely last month's report makes any sense today.
---
 frappe/hooks.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/frappe/hooks.py b/frappe/hooks.py
index 36a7748588..c5f19ebfb2 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -532,15 +532,15 @@ standard_help_items = [
 
 # log doctype cleanups to automatically add in log settings
 default_log_clearing_doctypes = {
-	"Error Log": 30,
-	"Activity Log": 90,
+	"Error Log": 14,
 	"Email Queue": 30,
-	"Scheduled Job Log": 90,
-	"Route History": 90,
-	"Submission Queue": 30,
-	"Prepared Report": 30,
+	"Scheduled Job Log": 7,
+	"Submission Queue": 7,
+	"Prepared Report": 14,
 	"Webhook Request Log": 30,
-	"Integration Request": 90,
 	"Unhandled Email": 30,
 	"Reminder": 30,
+	"Integration Request": 90,
+	"Activity Log": 90,
+	"Route History": 90,
 }

From 6341ef3ac70e078ac3d2c6a65171abfcdc929d2a Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 2 Feb 2024 18:31:03 +0530
Subject: [PATCH 56/64] fix: kanban hide "add column" for std fields (#24707)

* fix: build events

* fix(kanban): Don't show custom column option on standard fields
---
 .../public/js/frappe/build_events/build_events.bundle.js   | 2 +-
 .../public/js/frappe/views/kanban/kanban_board.bundle.js   | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/build_events/build_events.bundle.js b/frappe/public/js/frappe/build_events/build_events.bundle.js
index a191156bfa..5635502507 100644
--- a/frappe/public/js/frappe/build_events/build_events.bundle.js
+++ b/frappe/public/js/frappe/build_events/build_events.bundle.js
@@ -17,7 +17,7 @@ frappe.realtime.on("build_event", (data) => {
 					if (parts.length === 2) {
 						let filename = parts[0].split("/").slice(-1)[0];
 
-						frappe.assets.executed_ = frappe.assets.executed_.filter(
+						frappe.assets._executed = frappe.assets._executed.filter(
 							(asset) => !asset.includes(`${filename}.bundle`)
 						);
 					}
diff --git a/frappe/public/js/frappe/views/kanban/kanban_board.bundle.js b/frappe/public/js/frappe/views/kanban/kanban_board.bundle.js
index d3690571dd..bca9889859 100644
--- a/frappe/public/js/frappe/views/kanban/kanban_board.bundle.js
+++ b/frappe/public/js/frappe/views/kanban/kanban_board.bundle.js
@@ -377,8 +377,13 @@ frappe.provide("frappe.views");
 		}
 
 		function bind_add_column() {
-			if (!self.board_perms.write) {
+			let doctype = self.cur_list.doctype;
+			let fieldname = self.cur_list.board.field_name;
+			const is_custom_field = frappe.meta.get_docfield(doctype, fieldname)?.is_custom_field;
+
+			if (!self.board_perms.write || !is_custom_field) {
 				// If no write access to board, editing board (by adding column) should be blocked
+				// If standard field then users can't add options
 				self.$kanban_board.find(".add-new-column").remove();
 				return;
 			}

From 7487df22c9f48bedc58cb63a3a3818e633cd2c91 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 2 Feb 2024 18:43:43 +0530
Subject: [PATCH 57/64] refactor: use frappe.get_system_settings

because it's cached and doesn't hit frappe.db at all.
---
 frappe/auth.py                                 |  4 +---
 .../address_template/address_template.py       |  2 +-
 frappe/core/doctype/communication/mixins.py    |  2 +-
 frappe/core/doctype/user/user.py               | 14 +++++---------
 frappe/desk/notifications.py                   |  2 +-
 .../auto_email_report/auto_email_report.py     |  2 +-
 .../doctype/google_calendar/google_calendar.py |  2 +-
 frappe/twofactor.py                            | 18 ++++++++----------
 frappe/utils/password_strength.py              |  4 +---
 9 files changed, 20 insertions(+), 30 deletions(-)

diff --git a/frappe/auth.py b/frappe/auth.py
index 1e64e17ebb..521b0a6289 100644
--- a/frappe/auth.py
+++ b/frappe/auth.py
@@ -274,9 +274,7 @@ class LoginManager:
 		if self.user in frappe.STANDARD_USERS:
 			return False
 
-		reset_pwd_after_days = cint(
-			frappe.db.get_single_value("System Settings", "force_user_to_reset_password")
-		)
+		reset_pwd_after_days = cint(frappe.get_system_settings("force_user_to_reset_password"))
 
 		if reset_pwd_after_days:
 			last_password_reset_date = (
diff --git a/frappe/contacts/doctype/address_template/address_template.py b/frappe/contacts/doctype/address_template/address_template.py
index 601fb54302..2c9b27e585 100644
--- a/frappe/contacts/doctype/address_template/address_template.py
+++ b/frappe/contacts/doctype/address_template/address_template.py
@@ -28,7 +28,7 @@ class AddressTemplate(Document):
 
 		if not self.is_default and not self._get_previous_default():
 			self.is_default = 1
-			if frappe.db.get_single_value("System Settings", "setup_complete"):
+			if frappe.get_system_settings("setup_complete"):
 				frappe.msgprint(_("Setting this Address Template as default as there is no other default"))
 
 	def on_update(self):
diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py
index 2c05570cdb..222778f28e 100644
--- a/frappe/core/doctype/communication/mixins.py
+++ b/frappe/core/doctype/communication/mixins.py
@@ -146,7 +146,7 @@ class CommunicationEmailMixin:
 		return get_formatted_email(self.mail_sender_fullname(), mail=self.mail_sender())
 
 	def get_content(self, print_format=None):
-		if print_format and frappe.db.get_single_value("System Settings", "attach_view_link"):
+		if print_format and frappe.get_system_settings("attach_view_link"):
 			return self.content + self.get_attach_link(print_format)
 		return self.content
 
diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py
index aacad57e9a..ec95d54094 100644
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@@ -725,12 +725,8 @@ class User(Document):
 		3. If allow_login_using_user_name is set, you can use username while finding the user.
 		"""
 
-		login_with_mobile = cint(
-			frappe.db.get_single_value("System Settings", "allow_login_using_mobile_number")
-		)
-		login_with_username = cint(
-			frappe.db.get_single_value("System Settings", "allow_login_using_user_name")
-		)
+		login_with_mobile = cint(frappe.get_system_settings("allow_login_using_mobile_number"))
+		login_with_username = cint(frappe.get_system_settings("allow_login_using_user_name"))
 
 		or_filters = [{"name": user_name}]
 		if login_with_mobile:
@@ -840,8 +836,8 @@ def update_password(
 	else:
 		user = res["user"]
 
-	logout_all_sessions = cint(logout_all_sessions) or frappe.db.get_single_value(
-		"System Settings", "logout_on_password_reset"
+	logout_all_sessions = cint(logout_all_sessions) or frappe.get_system_settings(
+		"logout_on_password_reset"
 	)
 	_update_password(user, new_password, logout_all_sessions=cint(logout_all_sessions))
 
@@ -933,7 +929,7 @@ def _get_user_for_update_password(key, old_password):
 		result.user, last_reset_password_key_generated_on = user or (None, None)
 		if result.user:
 			reset_password_link_expiry = cint(
-				frappe.db.get_single_value("System Settings", "reset_password_link_expiry_duration")
+				frappe.get_system_settings("reset_password_link_expiry_duration")
 			)
 			if (
 				reset_password_link_expiry
diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py
index 4c728bdee9..090a3f6817 100644
--- a/frappe/desk/notifications.py
+++ b/frappe/desk/notifications.py
@@ -25,7 +25,7 @@ def get_notifications():
 		"open_count_doctype": {},
 		"targets": {},
 	}
-	if frappe.flags.in_install or not frappe.db.get_single_value("System Settings", "setup_complete"):
+	if frappe.flags.in_install or not frappe.get_system_settings("setup_complete"):
 		return out
 
 	config = get_notification_config()
diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index 6fe2596d7f..da93e7bb7e 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -92,7 +92,7 @@ class AutoEmailReport(Document):
 
 		max_reports_per_user = (
 			cint(frappe.local.conf.max_reports_per_user)  # kept for backward compatibilty
-			or cint(frappe.db.get_single_value("System Settings", "max_auto_email_report_per_user"))
+			or cint(frappe.get_system_settings("max_auto_email_report_per_user"))
 			or 20
 		)
 
diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py
index 8430e5c80c..2afd3222c5 100644
--- a/frappe/integrations/doctype/google_calendar/google_calendar.py
+++ b/frappe/integrations/doctype/google_calendar/google_calendar.py
@@ -239,7 +239,7 @@ def check_google_calendar(account, google_calendar):
 			# If no Calendar ID create a new Calendar
 			calendar = {
 				"summary": account.calendar_name,
-				"timeZone": frappe.db.get_single_value("System Settings", "time_zone"),
+				"timeZone": frappe.get_system_settings("time_zone"),
 			}
 			created_calendar = google_calendar.calendars().insert(body=calendar).execute()
 			frappe.db.set_value(
diff --git a/frappe/twofactor.py b/frappe/twofactor.py
index 6b23e3c0e4..a53ec27d06 100644
--- a/frappe/twofactor.py
+++ b/frappe/twofactor.py
@@ -43,11 +43,9 @@ def toggle_two_factor_auth(state, roles=None):
 
 def two_factor_is_enabled(user=None):
 	"""Return True if 2FA is enabled."""
-	enabled = cint(frappe.db.get_single_value("System Settings", "enable_two_factor_auth"))
+	enabled = cint(frappe.get_system_settings("enable_two_factor_auth"))
 	if enabled:
-		bypass_two_factor_auth = cint(
-			frappe.db.get_single_value("System Settings", "bypass_2fa_for_retricted_ip_users")
-		)
+		bypass_two_factor_auth = cint(frappe.get_system_settings("bypass_2fa_for_retricted_ip_users"))
 		if bypass_two_factor_auth and user:
 			user_doc = frappe.get_doc("User", user)
 			restrict_ip_list = (
@@ -144,7 +142,7 @@ def get_otpsecret_for_(user):
 
 
 def get_verification_method():
-	return frappe.db.get_single_value("System Settings", "two_factor_method")
+	return frappe.get_system_settings("two_factor_method")
 
 
 def confirm_otp_token(login_manager, otp=None, tmp_id=None):
@@ -190,7 +188,7 @@ def confirm_otp_token(login_manager, otp=None, tmp_id=None):
 
 
 def get_verification_obj(user, token, otp_secret):
-	otp_issuer = frappe.db.get_single_value("System Settings", "otp_issuer_name")
+	otp_issuer = frappe.get_system_settings("otp_issuer_name")
 	verification_method = get_verification_method()
 	verification_obj = None
 	if verification_method == "SMS":
@@ -263,7 +261,7 @@ def process_2fa_for_email(user, token, otp_secret, otp_issuer, method="Email"):
 def get_email_subject_for_2fa(kwargs_dict):
 	"""Get email subject for 2fa."""
 	subject_template = _("Login Verification Code from {}").format(
-		frappe.db.get_single_value("System Settings", "otp_issuer_name")
+		frappe.get_system_settings("otp_issuer_name")
 	)
 	return frappe.render_template(subject_template, kwargs_dict)
 
@@ -281,7 +279,7 @@ def get_email_body_for_2fa(kwargs_dict):
 def get_email_subject_for_qr_code(kwargs_dict):
 	"""Get QRCode email subject."""
 	subject_template = _("One Time Password (OTP) Registration Code from {}").format(
-		frappe.db.get_single_value("System Settings", "otp_issuer_name")
+		frappe.get_system_settings("otp_issuer_name")
 	)
 	return frappe.render_template(subject_template, kwargs_dict)
 
@@ -299,7 +297,7 @@ def get_link_for_qrcode(user, totp_uri):
 	key = frappe.generate_hash(length=20)
 	key_user = f"{key}_user"
 	key_uri = f"{key}_uri"
-	lifespan = int(frappe.db.get_single_value("System Settings", "lifespan_qrcode_image")) or 240
+	lifespan = int(frappe.get_system_settings("lifespan_qrcode_image")) or 240
 	frappe.cache.set_value(key_uri, totp_uri, expires_in_sec=lifespan)
 	frappe.cache.set_value(key_user, user, expires_in_sec=lifespan)
 	return get_url(f"/qrcode?k={key}")
@@ -433,7 +431,7 @@ def should_remove_barcode_image(barcode):
 	"""Check if it's time to delete barcode image from server."""
 	if isinstance(barcode, str):
 		barcode = frappe.get_doc("File", barcode)
-	lifespan = frappe.db.get_single_value("System Settings", "lifespan_qrcode_image") or 240
+	lifespan = frappe.get_system_settings("lifespan_qrcode_image") or 240
 	if time_diff_in_seconds(get_datetime(), barcode.creation) > int(lifespan):
 		return True
 	return False
diff --git a/frappe/utils/password_strength.py b/frappe/utils/password_strength.py
index 70ca375938..5b77eca4dd 100644
--- a/frappe/utils/password_strength.py
+++ b/frappe/utils/password_strength.py
@@ -50,9 +50,7 @@ default_feedback: "PasswordStrengthFeedback" = {
 def get_feedback(score: int, sequence: list) -> "PasswordStrengthFeedback":
 	"""Return the feedback dictionary consisting of ("warning","suggestions") for the given sequences."""
 	global default_feedback
-	minimum_password_score = int(
-		frappe.db.get_single_value("System Settings", "minimum_password_score") or 2
-	)
+	minimum_password_score = int(frappe.get_system_settings("minimum_password_score") or 2)
 
 	# Starting feedback
 	if len(sequence) == 0:

From 7580125bae0e94171655b3219136e5fd0ae7afbb Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 2 Feb 2024 18:57:47 +0530
Subject: [PATCH 58/64] fix: Filter auto repeat doctypes

---
 frappe/automation/doctype/auto_repeat/auto_repeat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py
index 956bb0c9c3..e8db131ebb 100644
--- a/frappe/automation/doctype/auto_repeat/auto_repeat.py
+++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py
@@ -550,7 +550,7 @@ def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters
 	docs += [r.name for r in res]
 	docs = set(list(docs))
 
-	return [[d] for d in docs]
+	return [[d] for d in docs if txt in d]
 
 
 @frappe.whitelist()

From 60c1812cc248c126971a60fbd0b9e905f4860ac0 Mon Sep 17 00:00:00 2001
From: Akhil Narang <me@akhilnarang.dev>
Date: Fri, 2 Feb 2024 22:26:01 +0530
Subject: [PATCH 59/64] fix(run-tests): don't try to access `frappe.flags`
 before `frappe.init()`

Broke in #24432

Signed-off-by: Akhil Narang <me@akhilnarang.dev>
---
 frappe/commands/utils.py | 5 ++---
 frappe/test_runner.py    | 5 +++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py
index a6eedf102a..333c37669f 100644
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@@ -772,9 +772,6 @@ def run_tests(
 			click.secho(f"bench --site {site} set-config allow_tests true", fg="green")
 			return
 
-		frappe.flags.skip_before_tests = skip_before_tests
-		frappe.flags.skip_test_records = skip_test_records
-
 		ret = frappe.test_runner.main(
 			site,
 			app,
@@ -789,6 +786,8 @@ def run_tests(
 			doctype_list_path=doctype_list_path,
 			failfast=failfast,
 			case=case,
+			skip_test_records=skip_test_records,
+			skip_before_tests=skip_before_tests,
 		)
 
 		if len(ret.failures) == 0 and len(ret.errors) == 0:
diff --git a/frappe/test_runner.py b/frappe/test_runner.py
index 9c072ae8e4..62e5dd599a 100644
--- a/frappe/test_runner.py
+++ b/frappe/test_runner.py
@@ -51,6 +51,8 @@ def main(
 	doctype_list_path=None,
 	failfast=False,
 	case=None,
+	skip_test_records=False,
+	skip_before_tests=False,
 ):
 	global unittest_runner
 
@@ -58,6 +60,9 @@ def main(
 	if not frappe.db:
 		frappe.connect()
 
+	frappe.flags.skip_before_tests = skip_before_tests
+	frappe.flags.skip_test_records = skip_test_records
+
 	if doctype_list_path:
 		app, doctype_list_path = doctype_list_path.split(os.path.sep, 1)
 		with open(frappe.get_app_path(app, doctype_list_path)) as f:

From 492584db93f9f9984d7af2d52e8b7b6dc5d64451 Mon Sep 17 00:00:00 2001
From: Govind Jangid <govindjangidsj@gmail.com>
Date: Sat, 3 Feb 2024 06:36:19 +0530
Subject: [PATCH 60/64] fix: date format & pre commit test

---
 .../auto_email_report/auto_email_report.py    | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index 2b82719232..fe2f25ca7b 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -2,9 +2,9 @@
 # License: MIT. See LICENSE
 
 import calendar
+import datetime
 from datetime import timedelta
 from email.utils import formataddr
-import datetime
 
 import frappe
 from frappe import _
@@ -15,18 +15,18 @@ from frappe.utils import (
 	add_to_date,
 	cint,
 	format_time,
+	get_first_day,
+	get_first_day_of_week,
 	get_link_to_form,
+	get_quarter_start,
 	get_url_to_report,
+	get_year_start,
+	getdate,
 	global_date_format,
 	now,
 	now_datetime,
 	today,
 	validate_email_address,
-	get_first_day_of_week,
-	get_first_day,
-	get_quarter_start,
-	get_year_start,
-	getdate,
 )
 from frappe.utils.csvutils import to_csv
 from frappe.utils.xlsxutils import make_xlsx
@@ -215,7 +215,7 @@ class AutoEmailReport(Document):
 		self.filters = frappe.parse_json(self.filters)
 
 		to_date = today()
-		
+
 		if self.use_first_day_of_period:
 			from_date = to_date
 			if self.dynamic_date_period == "Daily":
@@ -244,7 +244,7 @@ class AutoEmailReport(Document):
 
 			from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})
 			self.set_date_filters(from_date, to_date)
-		
+
 	def set_date_filters(self, from_date, to_date):
 		self.filters[self.from_date_field] = from_date
 		self.filters[self.to_date_field] = to_date
@@ -361,19 +361,22 @@ def update_field_types(columns):
 			col.options = ""
 	return columns
 
+
 DATE_FORMAT = "%Y-%m-%d"
+
+
 def get_half_year_start(as_str=False):
-    """
-    Returns the first day of the current half-year based on the current date.
-    """
-    today_date = getdate(today())
+	"""
+	Returns the first day of the current half-year based on the current date.
+	"""
+	today_date = getdate(today())
 
-    half_year = 1 if today_date.month <= 6 else 2
+	half_year = 1 if today_date.month <= 6 else 2
 
-    year = today_date.year if half_year == 1 else today_date.year + 1
-    month = 1 if half_year == 1 else 7
-    day = 1
+	year = today_date.year if half_year == 1 else today_date.year + 1
+	month = 1 if half_year == 1 else 7
+	day = 1
 
-    result_date = datetime.date(year, month, day)
+	result_date = datetime.date(year, month, day)
 
-    return result_date if not as_str else result_date.strftime(DATE_FORMAT)
+	return result_date if not as_str else result_date.strftime(DATE_FORMAT)

From f98af2e67af2e773080099e24f4aaefa6d8346d6 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Sat, 3 Feb 2024 12:35:09 +0530
Subject: [PATCH 61/64] fix: redirect and open new doctype dialog if route to
 /doctype/new

---
 frappe/public/js/form_builder/store.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/form_builder/store.js b/frappe/public/js/form_builder/store.js
index 91afafa707..7c0328362a 100644
--- a/frappe/public/js/form_builder/store.js
+++ b/frappe/public/js/form_builder/store.js
@@ -85,8 +85,13 @@ export const useStore = defineStore("form-builder-store", () => {
 
 	async function fetch() {
 		doc.value = frm.value.doc;
-		if (doctype.value.startsWith("new-doctype-") && !doc.value.fields) {
-			doc.value.fields = [get_df("Data", "", __("Title"))];
+		if (doctype.value.startsWith("new-doctype-") && !doc.value.fields?.length) {
+			frappe.model.with_doctype("DocType").then(() => {
+				frappe.listview_settings["DocType"].new_doctype_dialog();
+			});
+			// redirect to /doctype
+			frappe.set_route("List", "DocType");
+			return;
 		}
 
 		if (!get_docfields.value.length) {

From 0314502742ebc2677919959871564aa0cf4881e2 Mon Sep 17 00:00:00 2001
From: "Indrajith.vs" <91895505+Gubbu77@users.noreply.github.com>
Date: Sat, 3 Feb 2024 12:40:50 +0530
Subject: [PATCH 62/64] fix(Custom Field): Button Rename Fieldname visible
 before saving (#24712)

---
 .../doctype/custom_field/custom_field.js      | 46 ++++++++++---------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/frappe/custom/doctype/custom_field/custom_field.js b/frappe/custom/doctype/custom_field/custom_field.js
index 031d53de20..d0c61b6d8c 100644
--- a/frappe/custom/doctype/custom_field/custom_field.js
+++ b/frappe/custom/doctype/custom_field/custom_field.js
@@ -112,28 +112,30 @@ frappe.ui.form.on("Custom Field", {
 		}
 	},
 	add_rename_field(frm) {
-		frm.add_custom_button(__("Rename Fieldname"), () => {
-			frappe.prompt(
-				{
-					fieldtype: "Data",
-					label: __("Fieldname"),
-					fieldname: "fieldname",
-					reqd: 1,
-					default: frm.doc.fieldname,
-				},
-				function (data) {
-					frappe.call({
-						method: "frappe.custom.doctype.custom_field.custom_field.rename_fieldname",
-						args: {
-							custom_field: frm.doc.name,
-							fieldname: data.fieldname,
-						},
-					});
-				},
-				__("Rename Fieldname"),
-				__("Rename")
-			);
-		});
+		if (!frm.is_new()) {
+			frm.add_custom_button(__("Rename Fieldname"), () => {
+				frappe.prompt(
+					{
+						fieldtype: "Data",
+						label: __("Fieldname"),
+						fieldname: "fieldname",
+						reqd: 1,
+						default: frm.doc.fieldname,
+					},
+					function (data) {
+						frappe.call({
+							method: "frappe.custom.doctype.custom_field.custom_field.rename_fieldname",
+							args: {
+								custom_field: frm.doc.name,
+								fieldname: data.fieldname,
+							},
+						});
+					},
+					__("Rename Fieldname"),
+					__("Rename")
+				);
+			});
+		}
 	},
 });
 

From 25692a3c8192857d9d67b767078c473752a18cb2 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Sat, 3 Feb 2024 12:48:47 +0530
Subject: [PATCH 63/64] fix: Set default for search result limit if empty
 (#24713)

---
 frappe/core/doctype/system_settings/system_settings.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py
index db1511c893..013172a572 100644
--- a/frappe/core/doctype/system_settings/system_settings.py
+++ b/frappe/core/doctype/system_settings/system_settings.py
@@ -132,6 +132,9 @@ class SystemSettings(Document):
 		self.validate_backup_limit()
 		self.validate_file_extensions()
 
+		if not self.link_field_results_limit:
+			self.link_field_results_limit = 10
+
 		if self.link_field_results_limit > 50:
 			self.link_field_results_limit = 50
 			label = _(self.meta.get_label("link_field_results_limit"))

From cfd5b91b6ea86643ab226f0892a67bf2926b67a6 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Sat, 3 Feb 2024 14:32:55 +0530
Subject: [PATCH 64/64] fix: data import table UI fix

---
 .../public/js/frappe/data_import/import_preview.js   |  2 +-
 frappe/public/scss/desk/data_import.scss             | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js
index 4b0f18e3e1..2dcd22acf3 100644
--- a/frappe/public/js/frappe/data_import/import_preview.js
+++ b/frappe/public/js/frappe/data_import/import_preview.js
@@ -34,7 +34,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
 					<div class="col-sm-12">
 						<div class="table-actions margin-bottom">
 						</div>
-						<div class="table-preview border"></div>
+						<div class="table-preview"></div>
 						<div class="table-message"></div>
 					</div>
 				</div>
diff --git a/frappe/public/scss/desk/data_import.scss b/frappe/public/scss/desk/data_import.scss
index 2d40b82b71..4a82f22f06 100644
--- a/frappe/public/scss/desk/data_import.scss
+++ b/frappe/public/scss/desk/data_import.scss
@@ -14,4 +14,16 @@
 
 .table-preview {
 	margin-top: 12px;
+
+	.datatable .dt-scrollable .dt-row:last-child .dt-cell {
+		border-bottom: 1px solid var(--border-color);
+	}
+
+	.dt-row:last-child:not(.dt-row-filter) {
+		border-bottom: none;
+	}
+
+	.datatable .dt-header .dt-row-header {
+		background-color: unset;
+	}
 }