Merge branch 'develop' into attribution-page

.github/helper/install.sh

@@ -66,6 +66,6 @@ bench --site test_site reinstall --yes
 
 if [ "$TYPE" == "server" ]
 then
-  # wait till assets are built succesfully
+  # wait till assets are built successfully
   wait $build_pid
 fi

.github/helper/roulette.py

@@ -73,8 +73,9 @@ def has_label(pr_number, label, repo="frappe/frappe"):
 	)
 
 
-def is_py(file):
-	return file.endswith("py")
+def is_server_side_code(file):
+	"""File exclusively affects server side code"""
+	return file.endswith("py") or file.endswith(".po")
 
 
 def is_ci(file):
@@ -112,7 +113,7 @@ if __name__ == "__main__":
 	ci_files_changed = any(f for f in files_list if is_ci(f))
 	only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list)
 	only_frontend_code_changed = len(list(filter(is_frontend_code, files_list))) == len(files_list)
-	updated_py_file_count = len(list(filter(is_py, files_list)))
+	updated_py_file_count = len(list(filter(is_server_side_code, files_list)))
 	only_py_changed = updated_py_file_count == len(files_list)
 
 	if has_skip_ci_label(pr_number, repo):

.github/workflows/create-release.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
       - name: Setup dependencies
         run: |
           npm install @semantic-release/git @semantic-release/exec --no-save

.github/workflows/linters.yml

@@ -51,7 +51,7 @@ jobs:
           python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER
 
   linter:
-    name: 'Frappe Linter'
+    name: 'Semgrep Rules'
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
 
@@ -61,7 +61,6 @@ jobs:
         with:
           python-version: '3.10'
           cache: pip
-      - uses: pre-commit/action@v3.0.0
 
       - name: Download Semgrep rules
         run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules
@@ -83,7 +82,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
@@ -95,5 +94,4 @@ jobs:
         run: |
           pip install pip-audit
           cd ${GITHUB_WORKSPACE}
-          sed -i '/dropbox/d' pyproject.toml   # Remove dropbox temporarily https://github.com/dropbox/dropbox-sdk-python/pull/456
-          pip-audit --desc on --ignore-vuln GHSA-4xqq-73wg-5mjp .
+          pip-audit --desc on .

.github/workflows/patch-mariadb-tests.yml

@@ -76,7 +76,7 @@ jobs:
         run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
@@ -88,7 +88,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}

.github/workflows/pre-commit.yml

@@ -0,0 +1,26 @@
+name: Pre-commit
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: precommit-frappe-${{ github.event_name }}-${{ github.event.number }}
+  cancel-in-progress: true
+
+jobs:
+  linter:
+    name: 'precommit'
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: pip
+      - uses: pre-commit/action@v3.0.0

.github/workflows/server-tests.yml

@@ -104,7 +104,7 @@ jobs:
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
@@ -116,7 +116,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}

.github/workflows/ui-tests.yml

@@ -87,7 +87,7 @@ jobs:
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
@@ -99,7 +99,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -108,7 +108,7 @@ jobs:
             ${{ runner.os }}-yarn-ui-
 
       - name: Cache cypress binary
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/Cypress
           key: ${{ runner.os }}-cypress

.gitignore

@@ -169,6 +169,7 @@ typings/
 
 # Optional npm cache directory
 .npm
+.yarn
 
 # Optional eslint cache
 .eslintcache

babel_extractors.csv

@@ -0,0 +1,9 @@
+hooks.py,frappe.gettext.extractors.navbar.extract
+**/doctype/*/*.json,frappe.gettext.extractors.doctype.extract
+**/workspace/*/*.json,frappe.gettext.extractors.workspace.extract
+**/onboarding_step/*/*.json,frappe.gettext.extractors.onboarding_step.extract
+**/module_onboarding/*/*.json,frappe.gettext.extractors.module_onboarding.extract
+**/report/*/*.json,frappe.gettext.extractors.report.extract
+**.py,frappe.gettext.extractors.python.extract
+**.js,frappe.gettext.extractors.javascript.extract
+**.html,frappe.gettext.extractors.jinja2.extract

crowdin.yml

@@ -0,0 +1,3 @@
+files:
+  - source: /frappe/locale/main.pot
+    translation: /frappe/locale/%two_letters_code%.po

cypress/integration/form.js

@@ -7,7 +7,7 @@ const jump_to_field = (field_label) => {
 		.type("{enter}")
 		.wait(200)
 		.type("{enter}")
-		.wait(500);
+		.wait(1000);
 };
 
 const type_value = (value) => {

cypress/integration/list_view.js

@@ -25,6 +25,7 @@ context("List View", () => {
 			"Edit",
 			"Export",
 			"Assign To",
+			"Clear Assignment",
 			"Apply Assignment Rule",
 			"Add Tags",
 			"Print",
@@ -35,7 +36,7 @@ context("List View", () => {
 		cy.get(".list-header-subject > .list-subject > .list-check-all").click();
 		cy.findByRole("button", { name: "Actions" }).click();
 		cy.get(".dropdown-menu li:visible .dropdown-item")
-			.should("have.length", 9)
+			.should("have.length", 10)
 			.each((el, index) => {
 				cy.wrap(el).contains(actions[index]);
 			})

cypress/integration/routing.js

@@ -8,6 +8,7 @@ const test_queries = [
 	`?date=%5B">"%2C"2022-06-01"%5D`,
 	`?name=%5B"like"%2C"%2542%25"%5D`,
 	`?status=%5B"not%20in"%2C%5B"Open"%2C"Closed"%5D%5D`,
+	`?status=%5B%22%21%3D%22%2C%22Closed%22%5D&status=%5B%22%21%3D%22%2C%22Cancelled%22%5D`,
 ];
 
 describe("SPA Routing", { scrollBehavior: false }, () => {

cypress/support/commands.js

@@ -449,27 +449,8 @@ Cypress.Commands.add("click_menu_button", (name) => {
 });
 
 Cypress.Commands.add("clear_filters", () => {
-	let has_filter = false;
-	cy.intercept({
-		method: "POST",
-		url: "api/method/frappe.model.utils.user_settings.save",
-	}).as("filter-saved");
-	cy.get(".filter-section .filter-button").click({ force: true });
-	cy.wait(300);
-	cy.get(".filter-popover").should("exist");
-	cy.get(".filter-popover").then((popover) => {
-		if (popover.find("input.input-with-feedback")[0].value != "") {
-			has_filter = true;
-		}
-	});
-	cy.get(".filter-popover").find(".clear-filters").click();
-	cy.get(".filter-section .filter-button").click();
-	cy.window()
-		.its("cur_list")
-		.then((cur_list) => {
-			cur_list && cur_list.filter_area && cur_list.filter_area.clear();
-			has_filter && cy.wait("@filter-saved");
-		});
+	cy.get(".filter-x-button").click({ force: true });
+	cy.wait(500);
 });
 
 Cypress.Commands.add("click_modal_primary_button", (btn_name) => {

esbuild/esbuild.js

@@ -64,6 +64,11 @@ const argv = yargs
 		description:
 			"Saves esbuild metafiles for built assets. Useful for analyzing bundle size. More info: https://esbuild.github.io/api/#metafile",
 	})
+	.option("using-cached", {
+		type: "boolean",
+		description:
+			"Skips build and uses cached build artifacts to update assets.json (used by Bench)",
+	})
 	.example("node esbuild --apps frappe,erpnext", "Run build only for frappe and erpnext")
 	.example(
 		"node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js",
@@ -88,6 +93,7 @@ const NODE_PATHS = [].concat(
 	// import js file of any app if you provide the full path
 	app_list.map((app) => path.resolve(get_app_path(app), "..")).filter(fs.existsSync)
 );
+const USING_CACHED = Boolean(argv["using-cached"]);
 
 execute().catch((e) => {
 	console.error(e);
@@ -101,6 +107,12 @@ if (WATCH_MODE) {
 
 async function execute() {
 	console.time(TOTAL_BUILD_TIME);
+	if (USING_CACHED) {
+		await update_assets_json_from_built_assets(APPS);
+		await update_assets_json_in_cache();
+		console.timeEnd(TOTAL_BUILD_TIME);
+		process.exit(0);
+	}
 
 	let results;
 	try {
@@ -131,6 +143,44 @@ async function execute() {
 	}
 }
 
+async function update_assets_json_from_built_assets(apps) {
+	const assets = await get_assets_json_path_and_obj(false);
+	const assets_rtl = await get_assets_json_path_and_obj(true);
+
+	for (const app in apps) {
+		await update_assets_obj(app, assets.obj, assets_rtl.obj);
+	}
+
+	for (const { obj, path } of [assets, assets_rtl]) {
+		const data = JSON.stringify(obj, null, 4);
+		await fs.promises.writeFile(path, data);
+	}
+}
+
+async function update_assets_obj(app, assets, assets_rtl) {
+	const app_path = path.join(apps_path, app, app);
+	const dist_path = path.join(app_path, "public, dist");
+	const files = await glob("**/*.bundle.*.{js,css}", { cwd: dist_path });
+	const prefix = path.join("/", "assets", app, "dist");
+
+	// eg: "js/marketplace.bundle.6SCSPSGQ.js"
+	for (const file of files) {
+		// eg: [ "marketplace", "bundle", "6SCSPSGQ", "js" ]
+		const parts = path.parse(file).base.split(".");
+
+		// eg: "marketplace.bundle.js"
+		const key = [...parts.slice(0, -2), parts.at(-1)].join(".");
+
+		// eg: "js/marketplace.bundle.6SCSPSGQ.js"
+		const value = path.join(prefix, file);
+		if (file.includes("-rtl")) {
+			assets_rtl[`rtl_${key}`] = value;
+		} else {
+			assets[key] = value;
+		}
+	}
+}
+
 function build_assets_for_apps(apps, files) {
 	let { include_patterns, ignore_patterns } = files.length
 		? get_files_to_build(files)
@@ -393,14 +443,7 @@ async function write_assets_json(metafile) {
 		}
 	}
 
-	let assets_json_path = path.resolve(assets_path, `assets${rtl ? "-rtl" : ""}.json`);
-	let assets_json;
-	try {
-		assets_json = await fs.promises.readFile(assets_json_path, "utf-8");
-	} catch (error) {
-		assets_json = "{}";
-	}
-	assets_json = JSON.parse(assets_json);
+	let { obj: assets_json, path: assets_json_path } = await get_assets_json_path_and_obj(rtl);
 	// update with new values
 	let new_assets_json = Object.assign({}, assets_json, out);
 	curr_assets_json = new_assets_json;
@@ -434,6 +477,19 @@ async function update_assets_json_in_cache() {
 	});
 }
 
+async function get_assets_json_path_and_obj(is_rtl) {
+	const file_name = is_rtl ? "assets-rtl.json" : "assets.json";
+	const assets_json_path = path.resolve(assets_path, file_name);
+	let assets_json;
+	try {
+		assets_json = await fs.promises.readFile(assets_json_path, "utf-8");
+	} catch (error) {
+		assets_json = "{}";
+	}
+	assets_json = JSON.parse(assets_json);
+	return { obj: assets_json, path: assets_json_path };
+}
+
 function run_build_command_for_apps(apps) {
 	let cwd = process.cwd();
 	let { execSync } = require("child_process");

esbuild/utils.js

@@ -1,11 +1,16 @@
 const path = require("path");
 const fs = require("fs");
 const chalk = require("chalk");
+let bench_path;
+if (process.env.FRAPPE_BENCH_ROOT) {
+	bench_path = process.env.FRAPPE_BENCH_ROOT;
+} else {
+	const frappe_path = path.resolve(__dirname, "..");
+	bench_path = path.resolve(frappe_path, "..", "..");
+}
 
-const frappe_path = path.resolve(__dirname, "..");
-const bench_path = path.resolve(frappe_path, "..", "..");
-const sites_path = path.resolve(bench_path, "sites");
 const apps_path = path.resolve(bench_path, "apps");
+const sites_path = path.resolve(bench_path, "sites");
 const assets_path = path.resolve(sites_path, "assets");
 const app_list = get_apps_list();
 

frappe/__init__.py

@@ -118,6 +118,23 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str:
 	return translated_string or non_translated_string
 
 
+def _lt(msg: str, lang: str | None = None, context: str | None = None):
+	"""Lazily translate a string.
+
+
+	This function returns a "lazy string" which when casted to string via some operation applies
+	translation first before casting.
+
+	This is only useful for translating strings in global scope or anything that potentially runs
+	before `frappe.init()`
+
+	Note: Result is not guaranteed to equivalent to pure strings for all operations.
+	"""
+	from frappe.translate import LazyTranslate
+
+	return LazyTranslate(msg, lang, context)
+
+
 def as_unicode(text, encoding: str = "utf-8") -> str:
 	"""Convert to unicode if required."""
 	if isinstance(text, str):
@@ -185,8 +202,19 @@ if TYPE_CHECKING:  # pragma: no cover
 # end: static analysis hack
 
 
-def init(site: str, sites_path: str = ".", new_site: bool = False, force=False) -> None:
-	"""Initialize frappe for the current site. Reset thread locals `frappe.local`"""
+def init(
+	site: str, sites_path: str = ".", new_site: bool = False, force=False, site_ready: bool = True
+) -> None:
+	"""
+	Initialize frappe for the current site. Reset thread locals `frappe.local`
+
+	:param site: Site name.
+	:param sites_path: Path to sites directory.
+	:param new_site: Sets a flag to indicate a new site.
+	:param force: Force initialization if already previously run.
+	:param site_ready: Any init during site installation should set this to False.
+
+	"""
 	if getattr(local, "initialised", None) and not force:
 		return
 
@@ -244,34 +272,47 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False)
 	local.qb = get_query_builder(local.conf.db_type)
 	local.qb.get_query = get_query
 	setup_redis_cache_connection()
-	setup_module_map()
 
 	if not _qb_patched.get(local.conf.db_type):
 		patch_query_execute()
 		patch_query_aggregation()
 
+	if site:
+		setup_module_map(site_ready)
+
 	local.initialised = True
 
+	# Set the user as database name if not set in config
+	if local.conf and local.conf.db_name is not None and local.conf.db_user is None:
+		local.conf.db_user = local.conf.db_name
+
 
 def connect(
 	site: str | None = None, db_name: str | None = None, set_admin_as_user: bool = True
 ) -> None:
 	"""Connect to site database instance.
 
-	:param site: If site is given, calls `frappe.init`.
+	:param site: (Deprecated) If site is given, calls `frappe.init`.
 	:param db_name: Optional. Will use from `site_config.json`.
 	:param set_admin_as_user: Set Administrator as current user.
 	"""
 	from frappe.database import get_db
 
 	if site:
+		from frappe.utils.deprecations import deprecation_warning
+
+		deprecation_warning(
+			"Calling frappe.connect with the site argument is deprecated and will be removed in next major version. "
+			"Instead, explicitly invoke frappe.init(site) prior to calling frappe.connect(), if initializing the site is necessary."
+		)
 		init(site)
 
 	local.db = get_db(
 		host=local.conf.db_host,
 		port=local.conf.db_port,
-		user=db_name or local.conf.db_name,
-		password=None,
+		user=local.conf.db_user or db_name or local.conf.db_name,
+		password=local.conf.db_password,
+		cur_db_name=db_name or local.conf.db_name,
 	)
 	if set_admin_as_user:
 		set_user("Administrator")
@@ -283,15 +324,21 @@ def connect_replica() -> bool:
 	if local and hasattr(local, "replica_db") and hasattr(local, "primary_db"):
 		return False
 
-	user = local.conf.db_name
+	user = local.conf.db_user
 	password = local.conf.db_password
 	port = local.conf.replica_db_port
 
 	if local.conf.different_credentials_for_replica:
-		user = local.conf.replica_db_name
+		user = local.conf.replica_db_user or local.conf.replica_db_name
 		password = local.conf.replica_db_password
 
-	local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port)
+	local.replica_db = get_db(
+		host=local.conf.replica_host,
+		port=port,
+		user=user,
+		password=password,
+		cur_db_name=local.conf.db_name,
+	)
 
 	# swap db connections
 	local.primary_db = local.db
@@ -308,8 +355,10 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 	sites_path = sites_path or getattr(local, "sites_path", None)
 	site_path = site_path or getattr(local, "site_path", None)
 
+	common_config = get_common_site_config(sites_path)
+
 	if sites_path:
-		config.update(get_common_site_config(sites_path))
+		config.update(common_config)
 
 	if site_path:
 		site_config = os.path.join(site_path, "site_config.json")
@@ -320,7 +369,15 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 				click.secho(f"{local.site}/site_config.json is invalid", fg="red")
 				print(error)
 		elif local.site and not local.flags.new_site:
-			raise IncorrectSitePath(f"{local.site} does not exist")
+			error_msg = f"{local.site} does not exist."
+			if common_config.developer_mode:
+				from frappe.utils import get_sites
+
+				all_sites = get_sites()
+				error_msg += "\n\nSites on this bench:\n"
+				error_msg += "\n".join(f"* {site}" for site in all_sites)
+
+			raise IncorrectSitePath(error_msg)
 
 	# Generalized env variable overrides and defaults
 	def db_default_ports(db_type):
@@ -975,6 +1032,7 @@ def has_permission(
 	throw=False,
 	*,
 	parent_doctype=None,
+	debug=False,
 ):
 	"""
 	Return True if the user has permission `ptype` for given `doctype` or `doc`.
@@ -997,8 +1055,9 @@ def has_permission(
 		ptype,
 		doc=doc,
 		user=user,
-		raise_exception=throw,
+		print_logs=throw,
 		parent_doctype=parent_doctype,
+		debug=debug,
 	)
 
 	if throw and not out:
@@ -1590,18 +1649,32 @@ def append_hook(target, key, value):
 		target[key].extend(value)
 
 
-def setup_module_map():
-	"""Rebuild map of all modules (internal)."""
+def setup_module_map(site_ready: bool = True):
+	"""
+	Rebuild map of all modules (internal).
+
+	:param site_ready: If the site isn't fully ready yet - install is still going on, we can't
+	fetch apps from site DB. Fallback to fetching all apps on bench for module map temporarily.
+	"""
 	if conf.db_name:
 		local.app_modules = cache.get_value("app_modules")
 		local.module_app = cache.get_value("module_app")
 
 	if not (local.app_modules and local.module_app):
 		local.module_app, local.app_modules = {}, {}
-		for app in get_all_apps(with_internal_apps=True):
+
+		if site_ready:
+			apps = get_installed_apps(_ensure_on_bench=True)
+		else:
+			apps = get_all_apps()
+
+		for app in apps:
 			local.app_modules.setdefault(app, [])
 			for module in get_module_list(app):
 				module = scrub(module)
+				if module in local.module_app:
+					print(f"WARNING: module `{module}` found in apps `{local.module_app[module]}` and `{app}`")
+
 				local.module_app[module] = app
 				local.app_modules[app].append(module)
 

frappe/auth.py

@@ -4,13 +4,15 @@ import base64
 import binascii
 from urllib.parse import quote, urlencode, urlparse
 
+from werkzeug.wrappers import Response
+
 import frappe
 import frappe.database
 import frappe.utils
 import frappe.utils.user
 from frappe import _
 from frappe.core.doctype.activity_log.activity_log import add_authentication_log
-from frappe.sessions import Session, clear_sessions, delete_session
+from frappe.sessions import Session, clear_sessions, delete_session, get_expiry_in_seconds
 from frappe.translate import get_language
 from frappe.twofactor import (
 	authenticate_for_2factor,
@@ -356,12 +358,19 @@ class CookieManager:
 		if not frappe.local.session.get("sid"):
 			return
 
-		# sid expires in 3 days
-		expires = datetime.datetime.now() + datetime.timedelta(days=3)
 		if frappe.session.sid:
-			self.set_cookie("sid", frappe.session.sid, expires=expires, httponly=True)
+			self.set_cookie("sid", frappe.session.sid, max_age=get_expiry_in_seconds(), httponly=True)
 
-	def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"):
+	def set_cookie(
+		self,
+		key,
+		value,
+		expires=None,
+		secure=False,
+		httponly=False,
+		samesite="Lax",
+		max_age=None,
+	):
 		if not secure and hasattr(frappe.local, "request"):
 			secure = frappe.local.request.scheme == "https"
 
@@ -371,6 +380,7 @@ class CookieManager:
 			"secure": secure,
 			"httponly": httponly,
 			"samesite": samesite,
+			"max_age": max_age,
 		}
 
 	def delete_cookie(self, to_delete):
@@ -379,7 +389,7 @@ class CookieManager:
 
 		self.to_delete.extend(to_delete)
 
-	def flush_cookies(self, response):
+	def flush_cookies(self, response: Response):
 		for key, opts in self.cookies.items():
 			response.set_cookie(
 				key,
@@ -388,6 +398,7 @@ class CookieManager:
 				secure=opts.get("secure"),
 				httponly=opts.get("httponly"),
 				samesite=opts.get("samesite"),
+				max_age=opts.get("max_age"),
 			)
 
 		# expires yesterday!

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -9,6 +9,7 @@ from frappe.cache_manager import clear_doctype_map, get_doctype_map
 from frappe.desk.form import assign_to
 from frappe.model import log_types
 from frappe.model.document import Document
+from frappe.utils.data import comma_and
 
 
 class AssignmentRule(Document):
@@ -55,14 +56,10 @@ class AssignmentRule(Document):
 
 	def validate_assignment_days(self):
 		assignment_days = self.get_assignment_days()
-
 		if len(set(assignment_days)) != len(assignment_days):
-			repeated_days = get_repeated(assignment_days)
-			plural = "s" if len(repeated_days) > 1 else ""
-
 			frappe.throw(
-				_("Assignment Day{0} {1} has been repeated.").format(
-					plural, frappe.bold(", ".join(repeated_days))
+				_("The following Assignment Days have been repeated: {0}").format(
+					comma_and([_(day) for day in get_repeated(assignment_days)], add_quotes=False)
 				)
 			)
 

frappe/boot.py

@@ -237,7 +237,7 @@ def get_user_pages_or_reports(parent, cache=False):
 				has_role[p.name] = {"modified": p.modified, "title": p.title}
 
 	elif parent == "Report":
-		if not has_permission("Report", raise_exception=False):
+		if not has_permission("Report", print_logs=False):
 			return {}
 
 		reports = frappe.get_list(
@@ -270,9 +270,6 @@ def get_user_info():
 	user_info = frappe._dict()
 	add_user_info(frappe.session.user, user_info)
 
-	if frappe.session.user == "Administrator" and user_info.Administrator.email:
-		user_info[user_info.Administrator.email] = user_info.Administrator
-
 	return user_info
 
 

frappe/build.py

@@ -229,6 +229,7 @@ def bundle(
 	skip_frappe=False,
 	files=None,
 	save_metafiles=False,
+	using_cached=False,
 ):
 	"""concat / minify js files"""
 	setup()
@@ -246,7 +247,10 @@ def bundle(
 	if files:
 		command += " --files {files}".format(files=",".join(files))
 
-	command += " --run-build-command"
+	if using_cached:
+		command += " --using-cached"
+	else:
+		command += " --run-build-command"
 
 	if save_metafiles:
 		command += " --save-metafiles"

frappe/commands/__init__.py

@@ -105,6 +105,7 @@ def call_command(cmd, context):
 
 def get_commands():
 	# prevent circular imports
+	from .gettext import commands as gettext_commands
 	from .redis_utils import commands as redis_commands
 	from .scheduler import commands as scheduler_commands
 	from .site import commands as site_commands
@@ -113,7 +114,12 @@ def get_commands():
 
 	clickable_link = "https://frappeframework.com/docs"
 	all_commands = (
-		scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands
+		scheduler_commands
+		+ site_commands
+		+ translate_commands
+		+ gettext_commands
+		+ utils_commands
+		+ redis_commands
 	)
 
 	for command in all_commands:

frappe/commands/gettext.py

@@ -0,0 +1,99 @@
+import click
+
+from frappe.commands import pass_context
+from frappe.exceptions import SiteNotSpecifiedError
+
+
+@click.command("generate-pot-file", help="Translation: generate POT file")
+@click.option("--app", help="Only generate for this app. eg: frappe")
+@pass_context
+def generate_pot_file(context, app: str | None = None):
+	from frappe.gettext.translate import generate_pot
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	generate_pot(app)
+
+
+@click.command("compile-po-to-mo", help="Translation: compile PO files to MO files")
+@click.option("--app", help="Only compile for this app. eg: frappe")
+@click.option(
+	"--force",
+	is_flag=True,
+	default=False,
+	help="Force compile even if there are no changes to PO files",
+)
+@click.option("--locale", help="Compile transaltions only for this locale. eg: de")
+@pass_context
+def compile_translations(context, app: str | None = None, locale: str = None, force=False):
+	from frappe.gettext.translate import compile_translations as _compile_translations
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	_compile_translations(app, locale, force=force)
+
+
+@click.command(
+	"migrate-csv-to-po", help="Translation: migrate from CSV files (old) to PO files (new)"
+)
+@click.option("--app", help="Only migrate for this app. eg: frappe")
+@click.option("--locale", help="Compile translations only for this locale. eg: de")
+@pass_context
+def csv_to_po(context, app: str | None = None, locale: str = None):
+	from frappe.gettext.translate import migrate
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	migrate(app, locale)
+
+
+@click.command(
+	"update-po-files",
+	help="""Translation: sync PO files with POT file.
+You might want to run generate-pot-file first.""",
+)
+@click.option("--app", help="Only update for this app. eg: frappe")
+@click.option("--locale", help="Update PO files only for this locale. eg: de")
+@pass_context
+def update_po_files(context, app: str | None = None, locale: str | None = None):
+	from frappe.gettext.translate import update_po
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	update_po(app, locale=locale)
+
+
+@click.command("create-po-file", help="Translation: create a new PO file for a locale")
+@click.argument("locale", nargs=1)
+@click.option("--app", help="Only create for this app. eg: frappe")
+@pass_context
+def create_po_file(context, locale: str, app: str | None = None):
+	"""Create PO file for lang code"""
+	from frappe.gettext.translate import new_po
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	new_po(locale, app)
+
+
+def connect_to_site(site):
+	from frappe import connect
+
+	if not site:
+		raise SiteNotSpecifiedError
+
+	connect(site=site)
+
+
+commands = [
+	generate_pot_file,
+	compile_translations,
+	csv_to_po,
+	update_po_files,
+	create_po_file,
+]

frappe/commands/site.py

@@ -53,6 +53,7 @@ from frappe.exceptions import SiteNotSpecifiedError
 	default=True,
 	help="Create user and database in mariadb/postgres; only bootstrap if false",
 )
+@click.option("--db-user", help="Database user if you already have one")
 def new_site(
 	site,
 	db_root_username=None,
@@ -68,13 +69,14 @@ def new_site(
 	db_type=None,
 	db_host=None,
 	db_port=None,
+	db_user=None,
 	set_default=False,
 	setup_db=True,
 ):
 	"Create a new site"
 	from frappe.installer import _new_site
 
-	frappe.init(site=site, new_site=True)
+	frappe.init(site=site, new_site=True, site_ready=False)
 
 	_new_site(
 		db_name,
@@ -91,6 +93,7 @@ def new_site(
 		db_type=db_type,
 		db_host=db_host,
 		db_port=db_port,
+		db_user=db_user,
 		setup_db=setup_db,
 	)
 
@@ -260,8 +263,28 @@ def restore_backup(
 	admin_password,
 	force,
 ):
+	from pathlib import Path
+
 	from frappe.installer import _new_site, is_downgrade, is_partial, validate_database_sql
 
+	# Check for the backup file in the backup directory, as well as the main bench directory
+	dirs = (f"{site}/private/backups", "..")
+
+	# Try to resolve path to the file if we can't find it directly
+	if not Path(sql_file_path).exists():
+		click.secho(
+			f"File {sql_file_path} not found. Trying to check in alternative directories.", fg="yellow"
+		)
+		for dir in dirs:
+			potential_path = Path(dir) / Path(sql_file_path)
+			if potential_path.exists():
+				sql_file_path = str(potential_path.resolve())
+				click.secho(f"File {sql_file_path} found.", fg="green")
+				break
+		else:
+			click.secho(f"File {sql_file_path} not found.", fg="red")
+			sys.exit(1)
+
 	if is_partial(sql_file_path):
 		click.secho(
 			"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
@@ -299,7 +322,7 @@ def restore_backup(
 		)
 
 	except Exception as err:
-		print(err.args[1])
+		print(err)
 		sys.exit(1)
 
 
@@ -319,7 +342,7 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None):
 	site = get_site(context)
 	verbose = context.verbose or verbose
 	frappe.init(site=site)
-	frappe.connect(site=site)
+	frappe.connect()
 	err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True)
 	if err:
 		click.secho("Failed to detect type of backup file", fg="red")
@@ -394,7 +417,7 @@ def _reinstall(
 	if not yes:
 		click.confirm("This will wipe your database. Are you sure you want to reinstall?", abort=True)
 	try:
-		frappe.init(site=site)
+		frappe.init(site=site, site_ready=False)
 		frappe.connect()
 		frappe.clear_cache()
 		installed = frappe.get_installed_apps()
@@ -406,7 +429,7 @@ def _reinstall(
 			frappe.db.close()
 		frappe.destroy()
 
-	frappe.init(site=site)
+	frappe.init(site=site, site_ready=False)
 
 	_new_site(
 		frappe.conf.db_name,
@@ -515,7 +538,8 @@ def add_db_index(context, doctype, column):
 
 	columns = column  # correct naming
 	for site in context.sites:
-		frappe.connect(site=site)
+		frappe.init(site=site)
+		frappe.connect()
 		try:
 			frappe.db.add_index(doctype, columns)
 			if len(columns) == 1:
@@ -557,7 +581,8 @@ def describe_database_table(context, doctype, column):
 	import json
 
 	for site in context.sites:
-		frappe.connect(site=site)
+		frappe.init(site=site)
+		frappe.connect()
 		try:
 			data = _extract_table_stats(doctype, column)
 			# NOTE: Do not print anything else in this to avoid clobbering the output.
@@ -643,7 +668,8 @@ def add_system_manager(context, email, first_name, last_name, send_welcome_email
 	import frappe.utils.user
 
 	for site in context.sites:
-		frappe.connect(site=site)
+		frappe.init(site=site)
+		frappe.connect()
 		try:
 			frappe.utils.user.add_system_manager(email, first_name, last_name, send_welcome_email, password)
 			frappe.db.commit()
@@ -669,7 +695,8 @@ def add_user_for_sites(
 	import frappe.utils.user
 
 	for site in context.sites:
-		frappe.connect(site=site)
+		frappe.init(site=site)
+		frappe.connect()
 		try:
 			add_new_user(email, first_name, last_name, user_type, send_welcome_email, password, add_role)
 			frappe.db.commit()
@@ -699,7 +726,6 @@ def disable_user(context, email):
 @pass_context
 def migrate(context, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
-	from traceback_with_variables import activate_by_import
 
 	from frappe.migrate import SiteMigration
 
@@ -1038,7 +1064,11 @@ def _drop_site(
 			sys.exit(1)
 
 	click.secho("Dropping site database and user", fg="green")
-	drop_user_and_database(frappe.conf.db_name, db_root_username, db_root_password)
+
+	frappe.flags.root_login = db_root_username
+	frappe.flags.root_password = db_root_password
+
+	drop_user_and_database(frappe.conf.db_name, frappe.conf.db_user)
 
 	archived_sites_path = archived_sites_path or os.path.join(
 		frappe.utils.get_bench_path(), "archived", "sites"
@@ -1316,7 +1346,6 @@ def build_search_index(context):
 @click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table")
 @pass_context
 def clear_log_table(context, doctype, days, no_backup):
-
 	"""If any logtype table grows too large then clearing it with DELETE query
 	is not feasible in reasonable time. This command copies recent data to new
 	table and replaces current table with new smaller table.

frappe/commands/translate.py

@@ -34,7 +34,8 @@ def new_language(context, lang_code, app):
 		raise Exception("--site is required")
 
 	# init site
-	frappe.connect(site=context["sites"][0])
+	frappe.init(site=context["sites"][0])
+	frappe.connect()
 	frappe.translate.write_translations_file(app, lang_code)
 
 	print(

frappe/commands/utils.py

@@ -36,6 +36,13 @@ EXTRA_ARGS_CTX = {"ignore_unknown_options": True, "allow_extra_args": True}
 	default=False,
 	help="Saves esbuild metafiles for built assets. Useful for analyzing bundle size. More info: https://esbuild.github.io/api/#metafile",
 )
+@click.option(
+	"--using-cached",
+	is_flag=True,
+	default=False,
+	envvar="USING_CACHED",
+	help="Skips build and uses cached build artifacts (cache is set by Bench). Ignored if developer_mode enabled.",
+)
 def build(
 	app=None,
 	apps=None,
@@ -44,9 +51,11 @@ def build(
 	verbose=False,
 	force=False,
 	save_metafiles=False,
+	using_cached=False,
 ):
 	"Compile JS and CSS source files"
 	from frappe.build import bundle, download_frappe_assets
+	from frappe.gettext.translate import compile_translations
 	from frappe.utils.synchronization import filelock
 
 	frappe.init("")
@@ -68,6 +77,9 @@ def build(
 		if production:
 			mode = "production"
 
+		if development:
+			using_cached = False
+
 		bundle(
 			mode,
 			apps=apps,
@@ -75,8 +87,19 @@ def build(
 			verbose=verbose,
 			skip_frappe=skip_frappe,
 			save_metafiles=save_metafiles,
+			using_cached=using_cached,
 		)
 
+		if apps and isinstance(apps, str):
+			apps = apps.split(",")
+
+		if not apps:
+			apps = frappe.get_all_apps()
+
+		for app in apps:
+			print("Compiling translations for", app)
+			compile_translations(app, force=force)
+
 
 @click.command("watch")
 @click.option("--apps", help="Watch assets for specific apps")
@@ -93,14 +116,13 @@ def watch(apps=None):
 def clear_cache(context):
 	"Clear cache, doctype cache and defaults"
 	import frappe.sessions
-	from frappe.desk.notifications import clear_notifications
 	from frappe.website.utils import clear_website_cache
 
 	for site in context.sites:
 		try:
-			frappe.connect(site)
+			frappe.init(site=site)
+			frappe.connect()
 			frappe.clear_cache()
-			clear_notifications()
 			clear_website_cache()
 		finally:
 			frappe.destroy()
@@ -592,7 +614,7 @@ def console(context, autoreload=False):
 	all_apps = frappe.get_installed_apps()
 	failed_to_import = []
 
-	for app in all_apps:
+	for app in list(all_apps):
 		try:
 			locals()[app] = __import__(app)
 		except ModuleNotFoundError:

frappe/contacts/doctype/contact/test_contact.py

@@ -23,15 +23,15 @@ class TestContact(FrappeTestCase):
 
 	def test_check_default_phone_and_mobile(self):
 		phones = [
-			{"phone": "+91 0000000000", "is_primary_phone": 0, "is_primary_mobile_no": 0},
-			{"phone": "+91 0000000001", "is_primary_phone": 0, "is_primary_mobile_no": 0},
-			{"phone": "+91 0000000002", "is_primary_phone": 1, "is_primary_mobile_no": 0},
-			{"phone": "+91 0000000003", "is_primary_phone": 0, "is_primary_mobile_no": 1},
+			{"phone": "+91 0000000010", "is_primary_phone": 0, "is_primary_mobile_no": 0},
+			{"phone": "+91 0000000011", "is_primary_phone": 0, "is_primary_mobile_no": 0},
+			{"phone": "+91 0000000012", "is_primary_phone": 1, "is_primary_mobile_no": 0},
+			{"phone": "+91 0000000013", "is_primary_phone": 0, "is_primary_mobile_no": 1},
 		]
 		contact = create_contact("Phone", "Mr", phones=phones)
 
-		self.assertEqual(contact.phone, "+91 0000000002")
-		self.assertEqual(contact.mobile_no, "+91 0000000003")
+		self.assertEqual(contact.phone, "+91 0000000012")
+		self.assertEqual(contact.mobile_no, "+91 0000000013")
 
 	def test_get_full_name(self):
 		self.assertEqual(get_full_name(first="John"), "John")

frappe/contacts/doctype/contact/test_records.json

@@ -31,9 +31,9 @@
 		],
 		"phone_nos": [
 			{
-				"phone": "+91 0000000000",
+				"phone": "+91 0000000001",
 				"is_primary_phone": 1
 			}
 		]
 	}
-]
+]

frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py

@@ -76,7 +76,7 @@ def create_linked_contact(link_list, address):
 		}
 	)
 	contact.add_email("test_contact@example.com", is_primary=True)
-	contact.add_phone("+91 0000000000", is_primary_phone=True)
+	contact.add_phone("+91 0000000020", is_primary_phone=True)
 
 	for name in link_list:
 		contact.append("links", {"link_doctype": "Test Custom Doctype", "link_name": name})
@@ -105,7 +105,7 @@ class TestAddressesAndContacts(FrappeTestCase):
 				"_Test First Name",
 				"_Test Last Name",
 				"_Test Address-Billing",
-				"+91 0000000000",
+				"+91 0000000020",
 				"",
 				"test_contact@example.com",
 				1,

frappe/core/doctype/comment/comment.py

@@ -93,7 +93,7 @@ class Comment(Document):
 
 	def remove_comment_from_cache(self):
 		_comments = get_comments_from_parent(self)
-		for c in _comments:
+		for c in list(_comments):
 			if c.get("name") == self.name:
 				_comments.remove(c)
 

frappe/core/doctype/communication/communication.js

@@ -267,6 +267,7 @@ frappe.ui.form.on("Communication", {
 		$.extend(args, {
 			subject: __("Re: {0}", [frm.doc.subject]),
 			recipients: frm.doc.sender,
+			is_a_reply: true,
 		});
 
 		new frappe.views.CommunicationComposer(args);
@@ -278,6 +279,7 @@ frappe.ui.form.on("Communication", {
 			subject: __("Res: {0}", [frm.doc.subject]),
 			recipients: frm.doc.sender,
 			cc: frm.doc.cc,
+			is_a_reply: true,
 		});
 		new frappe.views.CommunicationComposer(args);
 	},
@@ -287,6 +289,7 @@ frappe.ui.form.on("Communication", {
 		$.extend(args, {
 			forward: true,
 			subject: __("Fw: {0}", [frm.doc.subject]),
+			is_a_reply: true,
 		});
 
 		new frappe.views.CommunicationComposer(args);

frappe/core/doctype/communication/communication.py

@@ -306,7 +306,7 @@ class Communication(Document, CommunicationEmailMixin):
 		emails = split_emails(emails) if isinstance(emails, str) else (emails or [])
 		if exclude_displayname:
 			return [email.lower() for email in {parse_addr(email)[1] for email in emails} if email]
-		return [email.lower() for email in set(emails) if email]
+		return [email for email in set(emails) if email]
 
 	def to_list(self, exclude_displayname=True):
 		"""Return `to` list."""
@@ -501,14 +501,17 @@ def on_doctype_update():
 	frappe.db.add_index("Communication", ["message_id(140)"])
 
 
-def has_permission(doc, ptype, user):
+def has_permission(doc, ptype, user=None, debug=False):
 	if ptype == "read":
 		if doc.reference_doctype == "Communication" and doc.reference_name == doc.name:
-			return
+			return True
 
 		if doc.reference_doctype and doc.reference_name:
-			if frappe.has_permission(doc.reference_doctype, ptype="read", doc=doc.reference_name):
-				return True
+			return frappe.has_permission(
+				doc.reference_doctype, ptype="read", doc=doc.reference_name, user=user, debug=debug
+			)
+
+	return True
 
 
 def get_permission_query_conditions_for_communication(user):

frappe/core/doctype/data_export/data_export.js

@@ -41,7 +41,7 @@ const can_export = (frm) => {
 	if (!doctype) {
 		frappe.msgprint(__("Please select the Document Type."));
 	} else if (!parent_multicheck_options.length) {
-		frappe.msgprint(__("Atleast one field of Parent Document Type is mandatory"));
+		frappe.msgprint(__("At least one field of Parent Document Type is mandatory"));
 	} else {
 		is_valid_form = true;
 	}
@@ -145,6 +145,12 @@ const get_doctypes = (parentdt) => {
 const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {
 	const fields = get_fields(doctype);
 
+	frappe.model.std_fields
+		.filter((df) => ["owner", "creation"].includes(df.fieldname))
+		.forEach((df) => {
+			fields.push(df);
+		});
+
 	const options = fields.map((df) => {
 		return {
 			label: df.label,

frappe/core/doctype/data_export/exporter.py

@@ -212,8 +212,23 @@ class DataExporter:
 		# build list of valid docfields
 		tablecolumns = []
 		table_name = "tab" + dt
+
 		for f in frappe.db.get_table_columns_description(table_name):
 			field = meta.get_field(f.name)
+			if f.name in ["owner", "creation"]:
+				std_field = next((x for x in frappe.model.std_fields if x["fieldname"] == f.name), None)
+				if std_field:
+					field = frappe._dict(
+						{
+							"fieldname": std_field.get("fieldname"),
+							"label": std_field.get("label"),
+							"fieldtype": std_field.get("fieldtype"),
+							"options": std_field.get("options"),
+							"idx": 0,
+							"parent": dt,
+						}
+					)
+
 			if field and (
 				(self.select_columns and f.name in self.select_columns[dt]) or not self.select_columns
 			):
@@ -404,7 +419,6 @@ class DataExporter:
 					)
 					for ci, child in enumerate(data_row.run(as_dict=True)):
 						self.add_data_row(rows, c["doctype"], c["parentfield"], child, ci)
-
 			for row in rows:
 				self.writer.writerow(row)
 

frappe/core/doctype/data_export/test_data_exporter.py

@@ -88,8 +88,8 @@ class TestDataExporter(FrappeTestCase):
 		self.assertEqual(frappe.response["type"], "csv")
 		self.assertEqual(frappe.response["doctype"], self.doctype_name)
 		self.assertTrue(frappe.response["result"])
-		self.assertIn('Child Title 1",50', frappe.response["result"])
-		self.assertIn('Child Title 2",51', frappe.response["result"])
+		self.assertRegex(frappe.response["result"], r"Child Title 1.*?,50")
+		self.assertRegex(frappe.response["result"], r"Child Title 2.*?,51")
 
 	def test_export_type(self):
 		for type in ["csv", "Excel"]:

frappe/core/doctype/doctype/doctype.py

@@ -4,13 +4,10 @@
 import copy
 import json
 import os
-
-# imports - standard imports
 import re
 import shutil
 from typing import TYPE_CHECKING, Union
 
-# imports - module imports
 import frappe
 from frappe import _
 from frappe.cache_manager import clear_controller_cache, clear_user_cache
@@ -204,7 +201,6 @@ class DocType(Document):
 		self.validate_document_type()
 		validate_fields(self)
 		self.check_indexing_for_dashboard_links()
-
 		if not self.istable:
 			validate_permissions(self)
 
@@ -234,6 +230,7 @@ class DocType(Document):
 			"DocPerm",
 			"Custom Field",
 			"Customize Form Field",
+			"Web Form Field",
 			"DocField",
 		]
 
@@ -1614,7 +1611,6 @@ def validate_fields(meta):
 
 		check_illegal_characters(d.fieldname)
 		check_invalid_fieldnames(meta.get("name"), d.fieldname)
-		check_unique_fieldname(meta.get("name"), d.fieldname)
 		check_fieldname_length(d.fieldname)
 		check_hidden_and_mandatory(meta.get("name"), d)
 		check_unique_and_text(meta.get("name"), d)
@@ -1624,6 +1620,7 @@ def validate_fields(meta):
 		validate_data_field_type(d)
 
 		if not frappe.flags.in_migrate:
+			check_unique_fieldname(meta.get("name"), d.fieldname)
 			check_link_table_options(meta.get("name"), d)
 			check_illegal_mandatory(meta.get("name"), d)
 			check_dynamic_link_options(d)

frappe/core/doctype/doctype/doctype_list.js

@@ -3,7 +3,19 @@ frappe.listview_settings["DocType"] = {
 		this.new_doctype_dialog();
 	},
 
-	new_doctype_dialog() {
+	new_doctype_dialog(args) {
+		const {
+			doctype_name = "",
+			doctype_module = "",
+			is_submittable = 0,
+			is_child = 0,
+			is_virtual = 0,
+			is_single = 0,
+			is_tree = 0,
+			is_custom = 0,
+			editable_grid = 1,
+		} = args || {};
+
 		let non_developer = frappe.session.user !== "Administrator" || !frappe.boot.developer_mode;
 		let fields = [
 			{
@@ -11,6 +23,7 @@ frappe.listview_settings["DocType"] = {
 				fieldname: "name",
 				fieldtype: "Data",
 				reqd: 1,
+				default: doctype_name,
 			},
 			{ fieldtype: "Column Break" },
 			{
@@ -19,6 +32,7 @@ frappe.listview_settings["DocType"] = {
 				fieldtype: "Link",
 				options: "Module Def",
 				reqd: 1,
+				default: doctype_module,
 			},
 			{ fieldtype: "Section Break" },
 			{
@@ -29,6 +43,7 @@ frappe.listview_settings["DocType"] = {
 					"Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended."
 				),
 				depends_on: "eval:!doc.istable && !doc.issingle",
+				default: is_submittable,
 			},
 			{
 				label: __("Is Child Table"),
@@ -36,13 +51,14 @@ frappe.listview_settings["DocType"] = {
 				fieldtype: "Check",
 				description: __("Child Tables are shown as a Grid in other DocTypes"),
 				depends_on: "eval:!doc.is_submittable && !doc.issingle",
+				default: is_child,
 			},
 			{
 				label: __("Editable Grid"),
 				fieldname: "editable_grid",
 				fieldtype: "Check",
 				depends_on: "istable",
-				default: 1,
+				default: editable_grid,
 			},
 			{
 				label: __("Is Single"),
@@ -52,12 +68,13 @@ frappe.listview_settings["DocType"] = {
 					"Single Types have only one record no tables associated. Values are stored in tabSingles"
 				),
 				depends_on: "eval:!doc.istable && !doc.is_submittable",
+				default: is_single,
 			},
 			{
 				label: "Is Tree",
 				fieldname: "is_tree",
 				fieldtype: "Check",
-				default: "0",
+				default: is_tree,
 				depends_on: "eval:!doc.istable",
 				description: "Tree structures are implemented using Nested Set",
 			},
@@ -65,7 +82,7 @@ frappe.listview_settings["DocType"] = {
 				label: __("Custom?"),
 				fieldname: "custom",
 				fieldtype: "Check",
-				default: non_developer,
+				default: non_developer || is_custom,
 				read_only: non_developer,
 			},
 		];
@@ -75,7 +92,7 @@ frappe.listview_settings["DocType"] = {
 				label: "Is Virtual",
 				fieldname: "is_virtual",
 				fieldtype: "Check",
-				default: "0",
+				default: is_virtual,
 			});
 		}
 

frappe/core/doctype/document_naming_settings/test_document_naming_settings.py

@@ -40,8 +40,8 @@ class TestNamingSeries(FrappeTestCase):
 
 	def get_valid_serieses(self):
 		VALID_SERIES = ["SINV-", "SI-.{field}.", "SI-#.###", ""]
-		exisiting_series = self.dns.get_transactions_and_prefixes()["prefixes"]
-		return VALID_SERIES + exisiting_series
+		existing_series = self.dns.get_transactions_and_prefixes()["prefixes"]
+		return VALID_SERIES + existing_series
 
 	def test_naming_preview(self):
 		self.dns.transaction_type = self.ns_doctype

frappe/core/doctype/file/file.py

@@ -778,11 +778,11 @@ def on_doctype_update():
 	frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"])
 
 
-def has_permission(doc, ptype=None, user=None):
+def has_permission(doc, ptype=None, user=None, debug=False):
 	user = user or frappe.session.user
 
 	if ptype == "create":
-		return frappe.has_permission("File", "create", user=user)
+		return frappe.has_permission("File", "create", user=user, debug=debug)
 
 	if not doc.is_private or (user != "Guest" and doc.owner == user) or user == "Administrator":
 		return True
@@ -798,9 +798,9 @@ def has_permission(doc, ptype=None, user=None):
 			return False
 
 		if ptype in ["write", "create", "delete"]:
-			return ref_doc.has_permission("write")
+			return ref_doc.has_permission("write", debug=debug, user=user)
 		else:
-			return ref_doc.has_permission("read")
+			return ref_doc.has_permission("read", debug=debug, user=user)
 
 	return False
 

frappe/core/doctype/permission_inspector/permission_inspector.js

@@ -0,0 +1,24 @@
+// Copyright (c) 2024, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+const call_debug = (frm) => {
+	frm.trigger("debug");
+};
+
+frappe.ui.form.on("Permission Inspector", {
+	refresh(frm) {
+		frm.disable_save();
+	},
+	docname: call_debug,
+	ref_doctype(frm) {
+		frm.doc.docname = ""; // Usually doctype change invalidates docname
+		call_debug(frm);
+	},
+	user: call_debug,
+	permission_type: call_debug,
+	debug(frm) {
+		if (frm.doc.ref_doctype && frm.doc.user) {
+			frm.call("debug");
+		}
+	},
+});

frappe/core/doctype/permission_inspector/permission_inspector.json

@@ -0,0 +1,90 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "beta": 1,
+ "creation": "2024-01-03 17:43:27.257317",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "ref_doctype",
+  "column_break_mcqo",
+  "docname",
+  "column_break_xbrd",
+  "user",
+  "column_break_nvaa",
+  "permission_type",
+  "section_break_hkjp",
+  "output"
+ ],
+ "fields": [
+  {
+   "fieldname": "ref_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "DocType",
+   "options": "DocType",
+   "reqd": 1
+  },
+  {
+   "fieldname": "docname",
+   "fieldtype": "Dynamic Link",
+   "in_list_view": 1,
+   "label": "Document",
+   "options": "ref_doctype"
+  },
+  {
+   "fieldname": "column_break_mcqo",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "column_break_xbrd",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "user",
+   "fieldtype": "Link",
+   "label": "User",
+   "options": "User",
+   "reqd": 1
+  },
+  {
+   "fieldname": "section_break_hkjp",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "output",
+   "fieldtype": "Code",
+   "label": "Output",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_nvaa",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "permission_type",
+   "fieldtype": "Select",
+   "label": "Permission Type",
+   "options": "read\nwrite\ncreate\ndelete\nsubmit\ncancel\nselect\namend\nprint\nemail\nreport\nimport\nexport\nshare"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "issingle": 1,
+ "links": [],
+ "modified": "2024-01-10 14:17:49.722593",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Permission Inspector",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "read": 1,
+   "role": "System Manager",
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/core/doctype/permission_inspector/permission_inspector.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+import frappe
+from frappe.model.document import Document
+from frappe.permissions import _pop_debug_log, has_permission
+
+
+class PermissionInspector(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		docname: DF.DynamicLink | None
+		output: DF.Code | None
+		permission_type: DF.Literal[
+			"read",
+			"write",
+			"create",
+			"delete",
+			"submit",
+			"cancel",
+			"select",
+			"amend",
+			"print",
+			"email",
+			"report",
+			"import",
+			"export",
+			"share",
+		]
+		ref_doctype: DF.Link
+		user: DF.Link
+	# end: auto-generated types
+
+	@frappe.whitelist()
+	def debug(self):
+		if not (self.ref_doctype and self.user):
+			return
+
+		result = has_permission(
+			self.ref_doctype, ptype=self.permission_type, doc=self.docname, user=self.user, debug=True
+		)
+
+		self.output = "\n==============================\n".join(_pop_debug_log())
+		self.output += "\n\n" + f"Ouput of has_permission: {result}"
+
+	# None of these apply, overriden for sanity.
+	def load_from_db(self):
+		super(Document, self).__init__({"modified": None, "permission_type": "read"})
+
+	def db_insert(self, *args, **kwargs):
+		...
+
+	def db_update(self):
+		...
+
+	@staticmethod
+	def get_list(args):
+		...
+
+	@staticmethod
+	def get_count(args):
+		...
+
+	@staticmethod
+	def get_stats(args):
+		...
+
+	def delete(self):
+		...

frappe/core/doctype/permission_inspector/test_permission_inspector.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2024, Frappe Technologies and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestPermissionInspector(FrappeTestCase):
+	pass

frappe/core/doctype/role_profile/role_profile.py

@@ -25,7 +25,7 @@ class RoleProfile(Document):
 		self.name = self.role_profile
 
 	def on_update(self):
-		self.queue_action("update_all_users", now=frappe.flags.in_test)
+		self.queue_action("update_all_users", now=frappe.flags.in_test, enqueue_after_commit=True)
 
 	def update_all_users(self):
 		"""Changes in role_profile reflected across all its user"""

frappe/core/doctype/rq_job/rq_job.json

@@ -2,7 +2,7 @@
  "actions": [],
  "allow_copy": 1,
  "autoname": "field:job_id",
- "creation": "2022-09-10 16:19:37.934903",
+ "creation": "2023-03-22 20:05:22.962044",
  "doctype": "DocType",
  "editable_grid": 1,
  "engine": "InnoDB",
@@ -104,10 +104,11 @@
    "fieldtype": "Section Break"
   }
  ],
+ "hide_toolbar": 1,
  "in_create": 1,
  "is_virtual": 1,
  "links": [],
- "modified": "2022-09-11 05:27:50.878534",
+ "modified": "2024-01-13 10:38:40.230972",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "RQ Job",

frappe/core/doctype/rq_job/rq_job.py

@@ -59,8 +59,8 @@ class RQJob(Document):
 		]
 		time_taken: DF.Duration | None
 		timeout: DF.Duration | None
-
 	# end: auto-generated types
+
 	def load_from_db(self):
 		try:
 			job = Job.fetch(self.name, connection=get_redis_conn())

frappe/core/doctype/rq_job/test_rq_job.py

@@ -166,7 +166,7 @@ class TestRQJob(FrappeTestCase):
 		# If this starts failing analyze memory usage using memray or some equivalent tool to find
 		# offending imports/function calls.
 		# Refer this PR: https://github.com/frappe/frappe/pull/21467
-		LAST_MEASURED_USAGE = 40
+		LAST_MEASURED_USAGE = 41
 		self.assertLessEqual(rss, LAST_MEASURED_USAGE * 1.05, msg)
 
 	@timeout(20)

frappe/core/doctype/rq_worker/rq_worker.json

@@ -110,10 +110,11 @@
    "label": "Utilization %"
   }
  ],
+ "hide_toolbar": 1,
  "in_create": 1,
  "is_virtual": 1,
  "links": [],
- "modified": "2022-11-24 14:50:48.511706",
+ "modified": "2024-01-13 10:36:13.034784",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "RQ Worker",

frappe/core/doctype/rq_worker/rq_worker.py

@@ -34,8 +34,8 @@ class RQWorker(Document):
 		total_working_time: DF.Duration | None
 		utilization_percent: DF.Percent
 		worker_name: DF.Data | None
-
 	# end: auto-generated types
+
 	def load_from_db(self):
 
 		all_workers = get_workers()

frappe/core/doctype/rq_worker/rq_worker_list.js

@@ -0,0 +1,9 @@
+frappe.listview_settings["RQ Worker"] = {
+	refresh(listview) {
+		listview.$no_result.html(`
+			<div class="no-result text-muted flex justify-center align-center">
+			${__("No RQ Workers connected. Try restarting the bench.")}
+			</div>
+		`);
+	},
+};

frappe/core/doctype/system_settings/system_settings.json

@@ -94,7 +94,9 @@
   "dormant_days",
   "telemetry_section",
   "allow_error_traceback",
-  "enable_telemetry"
+  "enable_telemetry",
+  "search_section",
+  "link_field_results_limit"
  ],
  "fields": [
   {
@@ -634,12 +636,24 @@
   {
    "fieldname": "column_break_uhqk",
    "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "search_section",
+   "fieldtype": "Section Break",
+   "label": "Search"
+  },
+  {
+   "default": "10",
+   "fieldname": "link_field_results_limit",
+   "fieldtype": "Int",
+   "label": "Link Field Results Limit",
+   "non_negative": 1
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2023-12-08 15:52:37.525003",
+ "modified": "2024-01-26 11:29:20.924425",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

frappe/core/doctype/system_settings/system_settings.py

@@ -61,6 +61,7 @@ class SystemSettings(Document):
 		hide_footer_in_auto_email_reports: DF.Check
 		language: DF.Link
 		lifespan_qrcode_image: DF.Int
+		link_field_results_limit: DF.Int
 		login_with_email_link: DF.Check
 		login_with_email_link_expiry: DF.Int
 		logout_on_password_reset: DF.Check
@@ -94,6 +95,7 @@ class SystemSettings(Document):
 		two_factor_method: DF.Literal["OTP App", "SMS", "Email"]
 		welcome_email_template: DF.Link | None
 	# end: auto-generated types
+
 	def validate(self):
 		from frappe.twofactor import toggle_two_factor_auth
 
@@ -130,6 +132,13 @@ class SystemSettings(Document):
 		self.validate_backup_limit()
 		self.validate_file_extensions()
 
+		if self.link_field_results_limit > 50:
+			self.link_field_results_limit = 50
+			label = _(self.meta.get_label("link_field_results_limit"))
+			frappe.msgprint(
+				_("{0} can not be more than {1}").format(label, 50), alert=True, indicator="yellow"
+			)
+
 	def validate_user_pass_login(self):
 		if not self.disable_user_pass_login:
 			return

frappe/core/doctype/translation/test_translation.py

@@ -3,7 +3,6 @@
 import frappe
 from frappe import _
 from frappe.tests.utils import FrappeTestCase
-from frappe.translate import clear_cache
 
 
 class TestTranslation(FrappeTestCase):
@@ -12,6 +11,8 @@ class TestTranslation(FrappeTestCase):
 
 	def tearDown(self):
 		frappe.local.lang = "en"
+		from frappe.translate import clear_cache
+
 		clear_cache()
 
 	def test_doctype(self):

frappe/core/doctype/user/user.json

@@ -462,7 +462,7 @@
    "read_only": 1
   },
   {
-   "default": "1",
+   "default": "2",
    "fieldname": "simultaneous_sessions",
    "fieldtype": "Int",
    "label": "Simultaneous Sessions"

frappe/core/doctype/user/user.py

@@ -1157,6 +1157,7 @@ def has_permission(doc, user):
 	if (user != "Administrator") and (doc.name in STANDARD_USERS):
 		# dont allow non Administrator user to view / edit Administrator user
 		return False
+	return True
 
 
 def notify_admin_access_to_system_manager(login_manager=None):

frappe/core/doctype/user_permission/user_permission_list.js

@@ -121,7 +121,7 @@ frappe.listview_settings["User Permission"] = {
 						callback: function (r) {
 							if (r.message === 1) {
 								frappe.show_alert({
-									message: __("User Permissions created sucessfully"),
+									message: __("User Permissions created successfully"),
 									indicator: "blue",
 								});
 							} else {

frappe/core/utils.py

@@ -8,7 +8,7 @@ import frappe
 
 def get_parent_doc(doc):
 	"""Return document of `reference_doctype`, `reference_doctype`."""
-	if not hasattr(doc, "parent_doc"):
+	if not getattr(doc, "parent_doc", None):
 		if doc.reference_doctype and doc.reference_name:
 			doc.parent_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name)
 		else:

frappe/core/workspace/build/build.json

@@ -13,21 +13,71 @@
  "label": "Build",
  "links": [
   {
+   "description": "Customize properties, naming, fields and more for standard doctypes",
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Models",
-   "link_count": 0,
+   "label": "Customization",
+   "link_count": 4,
    "link_type": "DocType",
    "onboard": 0,
-   "only_for": "",
    "type": "Card Break"
   },
   {
    "hidden": 0,
    "is_query_report": 0,
-   "label": "DocType",
+   "label": "Customize Form",
    "link_count": 0,
-   "link_to": "DocType",
+   "link_to": "Customize Form",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Custom Field",
+   "link_count": 0,
+   "link_to": "Custom Field",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Custom Translation",
+   "link_count": 0,
+   "link_to": "Translation",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Navbar Settings",
+   "link_count": 0,
+   "link_to": "Navbar Settings",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "description": "Group your custom doctypes under modules",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Modules",
+   "link_count": 2,
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Card Break"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Module Def",
+   "link_count": 0,
+   "link_to": "Module Def",
    "link_type": "DocType",
    "onboard": 0,
    "only_for": "",
@@ -36,22 +86,112 @@
   {
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Workflow",
+   "label": "Module Onboarding",
    "link_count": 0,
-   "link_to": "Workflow",
+   "link_to": "Module Onboarding",
    "link_type": "DocType",
    "onboard": 0,
    "only_for": "",
    "type": "Link"
   },
   {
+   "description": "Monitor logs for errors, background jobs, communications, and user activity",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "System Logs",
+   "link_count": 5,
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Card Break"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Background Jobs",
+   "link_count": 0,
+   "link_to": "RQ Job",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Scheduled Jobs Logs",
+   "link_count": 0,
+   "link_to": "Scheduled Job Log",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Error Logs",
+   "link_count": 0,
+   "link_to": "Error Log",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Communication Logs",
+   "link_count": 0,
+   "link_to": "Communication",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Activity Log",
+   "link_count": 0,
+   "link_to": "Activity Log",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "description": "Packages are lightweight apps (collection of Module Defs) that can be created, imported, or released right from the UI",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Packages",
+   "link_count": 2,
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Card Break"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Package",
+   "link_count": 0,
+   "link_to": "Package",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Package Import",
+   "link_count": 0,
+   "link_to": "Package Import",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "description": "Automate processes and extend standard functionality using scripts and background jobs",
    "hidden": 0,
    "is_query_report": 0,
    "label": "Scripting",
-   "link_count": 0,
+   "link_count": 3,
    "link_type": "DocType",
    "onboard": 0,
-   "only_for": "",
    "type": "Card Break"
   },
   {
@@ -88,38 +228,12 @@
    "type": "Link"
   },
   {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Packages",
-   "link_count": 2,
-   "onboard": 0,
-   "type": "Card Break"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Package",
-   "link_count": 0,
-   "link_to": "Package",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Package Import",
-   "link_count": 0,
-   "link_to": "Package Import",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
+   "description": "Build your own reports, print formats, and dashboards. Create personalized workspaces for easier navigation",
    "hidden": 0,
    "is_query_report": 0,
    "label": "Views",
    "link_count": 5,
+   "link_type": "DocType",
    "onboard": 0,
    "type": "Card Break"
   },
@@ -177,115 +291,10 @@
    "type": "Link"
   },
   {
+   "description": "Create new forms and views with doctypes. Set up multi-level workflows for approval",
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Customization",
-   "link_count": 4,
-   "onboard": 0,
-   "type": "Card Break"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Customize Form",
-   "link_count": 0,
-   "link_to": "Customize Form",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Custom Field",
-   "link_count": 0,
-   "link_to": "Custom Field",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Custom Translation",
-   "link_count": 0,
-   "link_to": "Translation",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Navbar Settings",
-   "link_count": 0,
-   "link_to": "Navbar Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "System Logs",
-   "link_count": 5,
-   "onboard": 0,
-   "type": "Card Break"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Background Jobs",
-   "link_count": 0,
-   "link_to": "RQ Job",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Scheduled Jobs Logs",
-   "link_count": 0,
-   "link_to": "Scheduled Job Log",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Error Logs",
-   "link_count": 0,
-   "link_to": "Error Log",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Communication Logs",
-   "link_count": 0,
-   "link_to": "Communication",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Activity Log",
-   "link_count": 0,
-   "link_to": "Activity Log",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Modules",
+   "label": "Models",
    "link_count": 2,
    "link_type": "DocType",
    "onboard": 0,
@@ -294,9 +303,9 @@
   {
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Module Def",
+   "label": "DocType",
    "link_count": 0,
-   "link_to": "Module Def",
+   "link_to": "DocType",
    "link_type": "DocType",
    "onboard": 0,
    "only_for": "",
@@ -305,16 +314,16 @@
   {
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Module Onboarding",
+   "label": "Workflow",
    "link_count": 0,
-   "link_to": "Module Onboarding",
+   "link_to": "Workflow",
    "link_type": "DocType",
    "onboard": 0,
    "only_for": "",
    "type": "Link"
   }
  ],
- "modified": "2024-01-02 15:38:42.806824",
+ "modified": "2024-01-23 17:27:44.769958",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Build",
@@ -325,7 +334,7 @@
  "quick_lists": [],
  "restrict_to_domain": "",
  "roles": [],
- "sequence_id": 16.0,
+ "sequence_id": 27.0,
  "shortcuts": [
   {
    "color": "Grey",

frappe/custom/doctype/custom_field/custom_field.js

@@ -119,6 +119,7 @@ frappe.ui.form.on("Custom Field", {
 					label: __("Fieldname"),
 					fieldname: "fieldname",
 					reqd: 1,
+					default: frm.doc.fieldname,
 				},
 				function (data) {
 					frappe.call({

frappe/custom/doctype/custom_field/custom_field.py

@@ -362,7 +362,8 @@ def rename_fieldname(custom_field: str, fieldname: str):
 		frappe.msgprint(_("Old and new fieldnames are same."), alert=True)
 		return
 
-	frappe.db.rename_column(parent_doctype, old_fieldname, new_fieldname)
+	if frappe.db.has_column(field.dt, old_fieldname):
+		frappe.db.rename_column(parent_doctype, old_fieldname, new_fieldname)
 
 	# Update in DB after alter column is successful, alter column will implicitly commit, so it's
 	# best to commit change on field too to avoid any possible mismatch between two.

frappe/database/__init__.py

@@ -36,34 +36,32 @@ def bootstrap_database(db_name, verbose=None, source_sql=None):
 		return frappe.database.mariadb.setup_db.bootstrap_database(db_name, verbose, source_sql)
 
 
-def drop_user_and_database(db_name, root_login=None, root_password=None):
+def drop_user_and_database(db_name, db_user):
 	import frappe
 
 	if frappe.conf.db_type == "postgres":
 		import frappe.database.postgres.setup_db
 
-		return frappe.database.postgres.setup_db.drop_user_and_database(
-			db_name, root_login, root_password
-		)
+		return frappe.database.postgres.setup_db.drop_user_and_database(db_name, db_user)
 	else:
 		import frappe.database.mariadb.setup_db
 
-		return frappe.database.mariadb.setup_db.drop_user_and_database(
-			db_name, root_login, root_password
-		)
+		return frappe.database.mariadb.setup_db.drop_user_and_database(db_name, db_user)
 
 
-def get_db(host=None, user=None, password=None, port=None):
+def get_db(host=None, user=None, password=None, port=None, cur_db_name=None):
 	import frappe
 
 	if frappe.conf.db_type == "postgres":
 		import frappe.database.postgres.database
 
-		return frappe.database.postgres.database.PostgresDatabase(host, user, password, port=port)
+		return frappe.database.postgres.database.PostgresDatabase(
+			host, user, password, port, cur_db_name
+		)
 	else:
 		import frappe.database.mariadb.database
 
-		return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port)
+		return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port, cur_db_name)
 
 
 def get_command(

frappe/database/database.py

@@ -10,7 +10,7 @@ import traceback
 from collections.abc import Iterable, Sequence
 from contextlib import contextmanager, suppress
 from time import time
-from typing import Any
+from typing import TYPE_CHECKING, Any, Union
 
 from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
 
@@ -34,11 +34,19 @@ from frappe.utils import cast as cast_fieldtype
 from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool
 from frappe.utils.deprecations import deprecation_warning
 
+if TYPE_CHECKING:
+	from psycopg2 import connection as PostgresConnection
+	from psycopg2 import cursor as PostgresCursor
+	from pymysql.connections import Connection as MariadbConnection
+	from pymysql.cursors import Cursor as MariadbCursor
+
 IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE)
 INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*")
 SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1')
 MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1')
 
+SQL_ITERATOR_BATCH_SIZE = 100
+
 
 class Database:
 	"""
@@ -65,27 +73,20 @@ class Database:
 		host=None,
 		user=None,
 		password=None,
-		ac_name=None,
-		use_default=0,
 		port=None,
+		cur_db_name=None,
 	):
 		self.setup_type_map()
-		self.host = host or frappe.conf.db_host
-		self.port = port or frappe.conf.db_port
-		self.user = user or frappe.conf.db_name
-		self.db_name = frappe.conf.db_name
+		self.host = host
+		self.port = port
+		self.user = user
+		self.password = password
+		self.cur_db_name = cur_db_name
 		self._conn = None
 
-		if ac_name:
-			self.user = ac_name or frappe.conf.db_name
-
-		if use_default:
-			self.user = frappe.conf.db_name
-
 		self.transaction_writes = 0
 		self.auto_commit_on_many_writes = 0
 
-		self.password = password or frappe.conf.db_password
 		self.value_cache = {}
 		self.logger = frappe.logger("database")
 		self.logger.setLevel("WARNING")
@@ -95,17 +96,16 @@ class Database:
 		self.before_rollback = CallbackManager()
 		self.after_rollback = CallbackManager()
 
-		# self.db_type: str
-		# self.last_query (lazy) attribute of last sql query executed
+	# self.db_type: str
+	# self.last_query (lazy) attribute of last sql query executed
 
 	def setup_type_map(self):
 		pass
 
 	def connect(self):
 		"""Connects to a database as set in `site_config.json`."""
-		self.cur_db_name = self.user
-		self._conn = self.get_connection()
-		self._cursor = self._conn.cursor()
+		self._conn: Union["MariadbConnection", "PostgresConnection"] = self.get_connection()
+		self._cursor: Union["MariadbCursor", "PostgresCursor"] = self._conn.cursor()
 
 		try:
 			if execution_timeout := get_query_execution_timeout():
@@ -121,6 +121,7 @@ class Database:
 	def use(self, db_name):
 		"""`USE` db_name."""
 		self._conn.select_db(db_name)
+		self.cur_db_name = db_name
 
 	def get_connection(self):
 		"""Return a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects."""
@@ -135,6 +136,9 @@ class Database:
 	def _transform_result(self, result: list[tuple]) -> list[tuple]:
 		return result
 
+	def _clean_up(self):
+		pass
+
 	def sql(
 		self,
 		query: Query,
@@ -149,6 +153,7 @@ class Database:
 		explain=False,
 		run=True,
 		pluck=False,
+		as_iterator=False,
 	):
 		"""Execute a SQL query and fetch all rows.
 
@@ -163,6 +168,9 @@ class Database:
 		:param run: Return query without executing it if False.
 		:param pluck: Get the plucked field only.
 		:param explain: Print `EXPLAIN` in error log.
+		:param as_iterator: Returns iterator over results instead of fetching all results at once.
+		        This should be used with unbuffered cursor as default cursors used by pymysql and postgres
+		        buffer the results internally. See `Database.unbuffered_cursor`.
 		Examples:
 
 		        # return customer names as dicts
@@ -264,21 +272,49 @@ class Database:
 		if not self._cursor.description:
 			return ()
 
-		self.last_result = self._transform_result(self._cursor.fetchall())
+		if as_iterator:
+			return self._return_as_iterator(pluck=pluck, as_dict=as_dict, as_list=as_list, update=update)
 
+		last_result = self._transform_result(self._cursor.fetchall())
 		if pluck:
-			return [r[0] for r in self.last_result]
+			last_result = [r[0] for r in last_result]
+			self._clean_up()
+			return last_result
 
 		# scrub output if required
 		if as_dict:
-			ret = self.fetch_as_dict()
+			last_result = self.fetch_as_dict(last_result)
 			if update:
-				for r in ret:
+				for r in last_result:
 					r.update(update)
-			return ret
+
 		elif as_list:
-			return self.convert_to_lists(self.last_result)
-		return self.last_result
+			last_result = self.convert_to_lists(last_result)
+
+		self._clean_up()
+		return last_result
+
+	def _return_as_iterator(self, *, pluck, as_dict, as_list, update):
+		while result := self._transform_result(self._cursor.fetchmany(SQL_ITERATOR_BATCH_SIZE)):
+			if pluck:
+				for row in result:
+					yield row[0]
+
+			elif as_dict:
+				keys = [column[0] for column in self._cursor.description]
+				for row in result:
+					row = frappe._dict(zip(keys, row))
+					if update:
+						row.update(update)
+					yield row
+
+			elif as_list:
+				for row in result:
+					yield list(row)
+			else:
+				frappe.throw(_("`as_iterator` only works with `as_list=True` or `as_dict=True`"))
+
+		self._clean_up()
 
 	def _log_query(
 		self,
@@ -396,9 +432,8 @@ class Database:
 		):
 			raise ImplicitCommitError("This statement can cause implicit commit", query)
 
-	def fetch_as_dict(self) -> list[frappe._dict]:
+	def fetch_as_dict(self, result) -> list[frappe._dict]:
 		"""Internal. Convert results to dict."""
-		result = self.last_result
 		if result:
 			keys = [column[0] for column in self._cursor.description]
 
@@ -437,6 +472,7 @@ class Database:
 		run=True,
 		pluck=False,
 		distinct=False,
+		skip_locked=False,
 	):
 		"""Return a document property or list of properties.
 
@@ -447,6 +483,10 @@ class Database:
 		:param as_dict: Return values as dict.
 		:param debug: Print query in error log.
 		:param order_by: Column to order by
+		:param cache: Use cached results fetched during current job/request
+		:param pluck: pluck first column instead of returning as nested list or dict.
+		:param for_update: All the affected/read rows will be locked.
+		:param skip_locked: Skip selecting currently locked rows.
 
 		Example:
 
@@ -477,6 +517,7 @@ class Database:
 			pluck=pluck,
 			distinct=distinct,
 			limit=1,
+			skip_locked=skip_locked,
 		)
 
 		if not run:
@@ -509,6 +550,7 @@ class Database:
 		pluck=False,
 		distinct=False,
 		limit=None,
+		skip_locked=False,
 	):
 		"""Return multiple document properties.
 
@@ -548,6 +590,8 @@ class Database:
 				distinct=distinct,
 				limit=limit,
 				as_dict=as_dict,
+				skip_locked=skip_locked,
+				for_update=for_update,
 			)
 
 		else:
@@ -568,11 +612,12 @@ class Database:
 						debug=debug,
 						order_by=order_by,
 						update=update,
-						for_update=for_update,
 						run=run,
 						pluck=pluck,
 						distinct=distinct,
 						limit=limit,
+						for_update=for_update,
+						skip_locked=skip_locked,
 					)
 				except Exception as e:
 					if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)):
@@ -731,7 +776,7 @@ class Database:
 		Example:
 
 		        # Update the `deny_multiple_sessions` field in System Settings DocType.
-		        company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True)
+		        frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True)
 		"""
 
 		to_update = self._get_update_dict(
@@ -806,6 +851,7 @@ class Database:
 		order_by=None,
 		update=None,
 		for_update=False,
+		skip_locked=False,
 		run=True,
 		pluck=False,
 		distinct=False,
@@ -816,6 +862,7 @@ class Database:
 			filters=filters,
 			order_by=order_by,
 			for_update=for_update,
+			skip_locked=skip_locked,
 			fields=fields,
 			distinct=distinct,
 			limit=limit,
@@ -839,6 +886,8 @@ class Database:
 		distinct=False,
 		limit=None,
 		as_dict=False,
+		for_update=False,
+		skip_locked=False,
 	):
 		if names := list(filter(None, names)):
 			return frappe.qb.get_query(
@@ -849,6 +898,8 @@ class Database:
 				distinct=distinct,
 				limit=limit,
 				validate_filters=True,
+				for_update=for_update,
+				skip_locked=skip_locked,
 			).run(debug=debug, run=run, as_dict=as_dict, pluck=pluck)
 		return {}
 
@@ -1288,6 +1339,22 @@ class Database:
 	def rename_column(self, doctype: str, old_column_name: str, new_column_name: str):
 		raise NotImplementedError
 
+	@contextmanager
+	def unbuffered_cursor(self):
+		"""Context manager to temporarily use unbuffered cursor.
+
+		Using this with `as_iterator=True` provides O(1) memory usage while reading large result sets.
+
+		NOTE: You MUST do entire result set processing in the context, otherwise underlying cursor
+		will be switched and you'll not get complete results.
+
+		Usage:
+		        with frappe.db.unbuffered_cursor():
+		                for row in frappe.db.sql("query with huge result", as_iterator=True):
+		                        continue # Do some processing.
+		"""
+		raise NotImplementedError
+
 
 @contextmanager
 def savepoint(catch: type | tuple[type, ...] = Exception):

frappe/database/db_manager.py

@@ -16,7 +16,7 @@ class DbManager:
 	def create_user(self, user, password, host=None):
 		host = host or self.get_current_host()
 		password_predicate = f" IDENTIFIED BY '{password}'" if password else ""
-		self.db.sql(f"CREATE USER '{user}'@'{host}'{password_predicate}")
+		self.db.sql(f"CREATE USER IF NOT EXISTS '{user}'@'{host}'{password_predicate}")
 
 	def delete_user(self, target, host=None):
 		host = host or self.get_current_host()
@@ -57,7 +57,7 @@ class DbManager:
 		from frappe.database import get_command
 		from frappe.utils import execute_in_shell
 
-		command = []
+		command = ["set -o pipefail;"]
 
 		if source.endswith(".gz"):
 			if gzip := which("gzip"):

frappe/database/mariadb/database.py

@@ -1,4 +1,5 @@
 import re
+from contextlib import contextmanager
 
 import pymysql
 from pymysql.constants import ER, FIELD_TYPE
@@ -123,8 +124,8 @@ class MariaDBConnectionUtil:
 			"use_unicode": True,
 		}
 
-		if self.user not in (frappe.flags.root_login, "root"):
-			conn_settings["database"] = self.user
+		if self.cur_db_name:
+			conn_settings["database"] = self.cur_db_name
 
 		if self.port:
 			conn_settings["port"] = int(self.port)
@@ -198,7 +199,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 			SUM(`data_length` + `index_length`) / 1024 / 1024 AS `database_size`
 			FROM information_schema.tables WHERE `table_schema` = %s GROUP BY `table_schema`
 			""",
-			self.db_name,
+			self.cur_db_name,
 			as_dict=True,
 		)
 
@@ -209,6 +210,13 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		self._log_query(self.last_query, debug, explain, query)
 		return self.last_query
 
+	def _clean_up(self):
+		# PERF: Erase internal references of pymysql to trigger GC as soon as
+		# results are consumed.
+		self._cursor._result = None
+		self._cursor._rows = None
+		self._cursor.connection._result = None
+
 	@staticmethod
 	def escape(s, percent=True):
 		"""Excape quotes and percent in given string."""
@@ -518,3 +526,15 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 
 		if est_row_size:
 			return int(est_row_size[0][0])
+
+	@contextmanager
+	def unbuffered_cursor(self):
+		from pymysql.cursors import SSCursor
+
+		try:
+			original_cursor = self._cursor
+			new_cursor = self._cursor = self._conn.cursor(SSCursor)
+			yield
+		finally:
+			self._cursor = original_cursor
+			new_cursor.close()

frappe/database/mariadb/setup_db.py

@@ -26,42 +26,45 @@ def get_mariadb_version(version_string: str = ""):
 def setup_database(force, verbose, no_mariadb_socket=False):
 	frappe.local.session = frappe._dict({"user": "Administrator"})
 
+	db_user = frappe.conf.db_user
 	db_name = frappe.local.conf.db_name
-	root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password)
+	root_conn = get_root_connection()
 	dbman = DbManager(root_conn)
 	dbman_kwargs = {}
 	if no_mariadb_socket:
 		dbman_kwargs["host"] = "%"
 
+	dbman.create_user(db_user, frappe.conf.db_password, **dbman_kwargs)
+	if verbose:
+		print(f"Created or updated user {db_user}")
+
 	if force or (db_name not in dbman.get_database_list()):
-		dbman.delete_user(db_name, **dbman_kwargs)
 		dbman.drop_database(db_name)
 	else:
 		raise Exception(f"Database {db_name} already exists")
 
-	dbman.create_user(db_name, frappe.conf.db_password, **dbman_kwargs)
-	if verbose:
-		print("Created user %s" % db_name)
-
 	dbman.create_database(db_name)
 	if verbose:
 		print("Created database %s" % db_name)
 
-	dbman.grant_all_privileges(db_name, db_name, **dbman_kwargs)
+	dbman.grant_all_privileges(db_name, db_user, **dbman_kwargs)
 	dbman.flush_privileges()
 	if verbose:
-		print(f"Granted privileges to user {db_name} and database {db_name}")
+		print(f"Granted privileges to user {db_user} and database {db_name}")
 
 	# close root connection
 	root_conn.close()
 
 
-def drop_user_and_database(db_name, root_login, root_password):
-	frappe.local.db = get_root_connection(root_login, root_password)
+def drop_user_and_database(
+	db_name,
+	db_user,
+):
+	frappe.local.db = get_root_connection()
 	dbman = DbManager(frappe.local.db)
 	dbman.drop_database(db_name)
-	dbman.delete_user(db_name, host="%")
-	dbman.delete_user(db_name)
+	dbman.delete_user(db_user, host="%")
+	dbman.delete_user(db_user)
 
 
 def bootstrap_database(db_name, verbose, source_sql=None):
@@ -96,14 +99,13 @@ def import_db_from_sql(source_sql=None, verbose=False):
 	if not source_sql:
 		source_sql = os.path.join(os.path.dirname(__file__), "framework_mariadb.sql")
 	DbManager(frappe.local.db).restore_database(
-		verbose, db_name, source_sql, db_name, frappe.conf.db_password
+		verbose, db_name, source_sql, frappe.conf.db_user, frappe.conf.db_password
 	)
 	if verbose:
 		print("Imported from database %s" % source_sql)
 
 
 def check_database_settings():
-
 	check_compatible_versions()
 
 	# Check each expected value vs. actuals:
@@ -152,24 +154,26 @@ def check_compatible_versions():
 		)
 
 
-def get_root_connection(root_login, root_password):
-	import getpass
-
+def get_root_connection():
 	if not frappe.local.flags.root_connection:
-		if not root_login:
-			root_login = "root"
+		from getpass import getpass
 
-		if not root_password:
-			root_password = frappe.conf.get("root_password") or None
+		if not frappe.flags.root_login:
+			frappe.flags.root_login = (
+				frappe.conf.get("root_login") or input("Enter mysql super user [root]: ") or "root"
+			)
 
-		if not root_password:
-			root_password = getpass.getpass("MySQL root password: ")
+		if not frappe.flags.root_password:
+			frappe.flags.root_password = frappe.conf.get("root_password") or getpass(
+				"MySQL root password: "
+			)
 
 		frappe.local.flags.root_connection = frappe.database.get_db(
 			host=frappe.conf.db_host,
 			port=frappe.conf.db_port,
-			user=root_login,
-			password=root_password,
+			user=frappe.flags.root_login,
+			password=frappe.flags.root_password,
+			cur_db_name=None,
 		)
 
 	return frappe.local.flags.root_connection

frappe/database/postgres/database.py

@@ -161,8 +161,8 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 
 	def get_connection(self):
 		conn_settings = {
+			"dbname": self.cur_db_name,
 			"user": self.user,
-			"dbname": self.user,
 			"host": self.host,
 			"password": self.password,
 		}
@@ -199,7 +199,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 	def get_database_size(self):
 		"""Return database size in MB"""
 		db_size = self.sql(
-			"SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", self.db_name, as_dict=True
+			"SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", self.cur_db_name, as_dict=True
 		)
 		return db_size[0].get("database_size")
 
@@ -219,7 +219,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 			where table_catalog='{}'
 				and table_type = 'BASE TABLE'
 				and table_schema='{}'""".format(
-					frappe.conf.db_name, frappe.conf.get("db_schema", "public")
+					self.cur_db_name, frappe.conf.get("db_schema", "public")
 				)
 			)
 		]

frappe/database/postgres/setup_db.py

@@ -7,19 +7,25 @@ from frappe.utils import cint
 
 
 def setup_database():
-	root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password)
+	root_conn = get_root_connection()
 	root_conn.commit()
 	root_conn.sql("end")
-	root_conn.sql(f"DROP DATABASE IF EXISTS `{frappe.conf.db_name}`")
-	root_conn.sql(f"DROP USER IF EXISTS {frappe.conf.db_name}")
-	root_conn.sql(f"CREATE DATABASE `{frappe.conf.db_name}`")
-	root_conn.sql(f"CREATE user {frappe.conf.db_name} password '{frappe.conf.db_password}'")
-	root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name))
+	root_conn.sql(f'DROP DATABASE IF EXISTS "{frappe.conf.db_name}"')
+
+	# If user exists, just update password
+	if root_conn.sql(f"SELECT 1 FROM pg_roles WHERE rolname='{frappe.conf.db_user}'"):
+		root_conn.sql(f"ALTER USER \"{frappe.conf.db_user}\" WITH PASSWORD '{frappe.conf.db_password}'")
+	else:
+		root_conn.sql(f"CREATE USER \"{frappe.conf.db_user}\" WITH PASSWORD '{frappe.conf.db_password}'")
+	root_conn.sql(f'CREATE DATABASE "{frappe.conf.db_name}"')
+	root_conn.sql(
+		f'GRANT ALL PRIVILEGES ON DATABASE "{frappe.conf.db_name}" TO "{frappe.conf.db_user}"'
+	)
 	if psql_version := root_conn.sql("SELECT VERSION()", as_dict=True):
 		version_string = psql_version[0].get("version") or "PostgreSQL 14"
 		major_version = cint(re.split(r"[\w\.]", version_string)[1])
 		if major_version > 15:
-			root_conn.sql("ALTER DATABASE `{0}` OWNER TO {0}".format(frappe.conf.db_name))
+			root_conn.sql(f'ALTER DATABASE "{frappe.conf.db_name}" OWNER TO "{frappe.conf.db_user}"')
 	root_conn.close()
 
 
@@ -49,42 +55,39 @@ def import_db_from_sql(source_sql=None, verbose=False):
 	if not source_sql:
 		source_sql = os.path.join(os.path.dirname(__file__), "framework_postgres.sql")
 	DbManager(frappe.local.db).restore_database(
-		verbose, db_name, source_sql, db_name, frappe.conf.db_password
+		verbose, db_name, source_sql, frappe.conf.db_user, frappe.conf.db_password
 	)
 	if verbose:
 		print("Imported from database %s" % source_sql)
 
 
-def get_root_connection(root_login=None, root_password=None):
+def get_root_connection():
 	if not frappe.local.flags.root_connection:
-		if not root_login:
-			root_login = frappe.conf.get("root_login") or None
+		from getpass import getpass
 
-		if not root_login:
-			root_login = input("Enter postgres super user: ")
+		if not frappe.flags.root_login:
+			frappe.flags.root_login = (
+				frappe.conf.get("root_login") or input("Enter postgres super user [postgres]: ") or "postgres"
+			)
 
-		if not root_password:
-			root_password = frappe.conf.get("root_password") or None
-
-		if not root_password:
-			from getpass import getpass
-
-			root_password = getpass("Postgres super user password: ")
+		if not frappe.flags.root_password:
+			frappe.flags.root_password = frappe.conf.get("root_password") or getpass(
+				"Postgres super user password: "
+			)
 
 		frappe.local.flags.root_connection = frappe.database.get_db(
 			host=frappe.conf.db_host,
 			port=frappe.conf.db_port,
-			user=root_login,
-			password=root_password,
+			user=frappe.flags.root_login,
+			password=frappe.flags.root_password,
+			cur_db_name=frappe.flags.root_login,
 		)
 
 	return frappe.local.flags.root_connection
 
 
-def drop_user_and_database(db_name, root_login, root_password):
-	root_conn = get_root_connection(
-		frappe.flags.root_login or root_login, frappe.flags.root_password or root_password
-	)
+def drop_user_and_database(db_name, db_user):
+	root_conn = get_root_connection()
 	root_conn.commit()
 	root_conn.sql(
 		"SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = %s",
@@ -92,4 +95,4 @@ def drop_user_and_database(db_name, root_login, root_password):
 	)
 	root_conn.sql("end")
 	root_conn.sql(f"DROP DATABASE IF EXISTS {db_name}")
-	root_conn.sql(f"DROP USER IF EXISTS {db_name}")
+	root_conn.sql(f"DROP USER IF EXISTS {db_user}")

frappe/database/query.py

@@ -47,6 +47,7 @@ class Engine:
 		delete: bool = False,
 		*,
 		validate_filters: bool = False,
+		skip_locked: bool = False,
 	) -> QueryBuilder:
 		self.is_mariadb = frappe.db.db_type == "mariadb"
 		self.is_postgres = frappe.db.db_type == "postgres"
@@ -83,7 +84,7 @@ class Engine:
 			self.query = self.query.distinct()
 
 		if for_update:
-			self.query = self.query.for_update()
+			self.query = self.query.for_update(skip_locked=skip_locked)
 
 		if group_by:
 			self.query = self.query.groupby(group_by)

frappe/desk/desktop.py

@@ -417,8 +417,11 @@ def get_workspace_sidebar_items():
 	blocked_modules = frappe.get_doc("User", frappe.session.user).get_blocked_modules()
 	blocked_modules.append("Dummy Module")
 
+	# adding None to allowed_domains to include pages without domain restriction
+	allowed_domains = [None] + frappe.get_active_domains()
+
 	filters = {
-		"restrict_to_domain": ["in", frappe.get_active_domains()],
+		"restrict_to_domain": ["in", allowed_domains],
 		"module": ["not in", blocked_modules],
 	}
 

frappe/desk/doctype/dashboard_chart/dashboard_chart.py

@@ -109,7 +109,7 @@ def get(
 	refresh=None,
 ):
 	if chart_name:
-		chart = frappe.get_doc("Dashboard Chart", chart_name)
+		chart: DashboardChart = frappe.get_doc("Dashboard Chart", chart_name)
 	else:
 		chart = frappe._dict(frappe.parse_json(chart))
 
@@ -207,13 +207,14 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 	filters.append([doctype, datefield, ">=", from_date, False])
 	filters.append([doctype, datefield, "<=", to_date, False])
 
-	data = frappe.db.get_list(
+	data = frappe.get_list(
 		doctype,
 		fields=[datefield, f"SUM({value_field})", "COUNT(*)"],
 		filters=filters,
 		group_by=datefield,
 		order_by=datefield,
 		as_list=True,
+		parent_doctype=chart.parent_document_type,
 	)
 
 	result = get_result(data, timegrain, from_date, to_date, chart.chart_type)

frappe/desk/doctype/desktop_icon/desktop_icon.py

@@ -271,7 +271,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True):
 		frappe.db.sql("update `tabDesktop Icon` set blocked=0, hidden=1 where standard=1")
 
 	# set as visible if present, or add icon
-	for module_name in visible_list:
+	for module_name in list(visible_list):
 		name = frappe.db.get_value("Desktop Icon", {"module_name": module_name})
 		if name:
 			frappe.db.set_value("Desktop Icon", name, "hidden", 0)

frappe/desk/doctype/event/event.js

@@ -44,9 +44,13 @@ frappe.ui.form.on("Event", {
 
 		const [ends_on_date] = frm.doc.ends_on
 			? frm.doc.ends_on.split(" ")
-			: frm.doc.starts_on.split(" ");
+			: frm.doc.starts_on?.split(" ") || [];
 
-		if (frm.doc.google_meet_link && frappe.datetime.now_date() <= ends_on_date) {
+		if (
+			ends_on_date &&
+			frm.doc.google_meet_link &&
+			frappe.datetime.now_date() <= ends_on_date
+		) {
 			frm.dashboard.set_headline(
 				__("Join video conference with {0}", [
 					`<a target='_blank' href='${frm.doc.google_meet_link}'>Google Meet</a>`,

frappe/desk/doctype/event/event.json

@@ -123,7 +123,7 @@
    "fieldtype": "Select",
    "in_global_search": 1,
    "label": "Repeat On",
-   "options": "\nDaily\nWeekly\nMonthly\nYearly"
+   "options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly"
   },
   {
    "depends_on": "repeat_this_event",
@@ -295,7 +295,7 @@
  "icon": "fa fa-calendar",
  "idx": 1,
  "links": [],
- "modified": "2023-06-23 10:33:15.685368",
+ "modified": "2024-01-11 07:11:17.467503",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Event",
@@ -336,4 +336,4 @@
  "track_changes": 1,
  "track_seen": 1,
  "track_views": 1
-}
+}

frappe/desk/doctype/event/event.py

@@ -21,6 +21,7 @@ from frappe.utils import (
 	format_datetime,
 	get_datetime_str,
 	getdate,
+	month_diff,
 	now_datetime,
 	nowdate,
 )
@@ -62,7 +63,7 @@ class Event(Document):
 		google_meet_link: DF.Data | None
 		monday: DF.Check
 		pulled_from_google_calendar: DF.Check
-		repeat_on: DF.Literal["", "Daily", "Weekly", "Monthly", "Yearly"]
+		repeat_on: DF.Literal["", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"]
 		repeat_this_event: DF.Check
 		repeat_till: DF.Date | None
 		saturday: DF.Check
@@ -392,6 +393,62 @@ def get_events(start, end, user=None, for_reminder=False, filters=None) -> list[
 
 				remove_events.append(e)
 
+			if e.repeat_on == "Half Yearly":
+				# creates a string with date (27) and month (07) and year (2019) eg: 2019-07-27
+				year, month = start.split("-", maxsplit=2)[:2]
+				date = f"{year}-{month}-" + event_start.split("-", maxsplit=3)[2]
+
+				# last day of month issue, start from prev month!
+				try:
+					getdate(date)
+				except Exception:
+					date = date.split("-")
+					date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2]
+
+				start_from = date
+				for i in range(int(date_diff(end, start) / 30) + 3):
+					diff = month_diff(date, event_start) - 1
+					if diff % 6 != 0:
+						continue
+					if (
+						getdate(date) >= getdate(start)
+						and getdate(date) <= getdate(end)
+						and getdate(date) <= getdate(repeat)
+						and getdate(date) >= getdate(event_start)
+					):
+						add_event(e, date)
+
+					date = add_months(start_from, i + 1)
+				remove_events.append(e)
+
+			if e.repeat_on == "Quarterly":
+				# creates a string with date (27) and month (07) and year (2019) eg: 2019-07-27
+				year, month = start.split("-", maxsplit=2)[:2]
+				date = f"{year}-{month}-" + event_start.split("-", maxsplit=3)[2]
+
+				# last day of month issue, start from prev month!
+				try:
+					getdate(date)
+				except Exception:
+					date = date.split("-")
+					date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2]
+
+				start_from = date
+				for i in range(int(date_diff(end, start) / 30) + 3):
+					diff = month_diff(date, event_start) - 1
+					if diff % 3 != 0:
+						continue
+					if (
+						getdate(date) >= getdate(start)
+						and getdate(date) <= getdate(end)
+						and getdate(date) <= getdate(repeat)
+						and getdate(date) >= getdate(event_start)
+					):
+						add_event(e, date)
+
+					date = add_months(start_from, i + 1)
+				remove_events.append(e)
+
 			if e.repeat_on == "Monthly":
 				# creates a string with date (27) and month (07) and year (2019) eg: 2019-07-27
 				year, month = start.split("-", maxsplit=2)[:2]

frappe/desk/doctype/event/test_event.py

@@ -136,3 +136,77 @@ class TestEvent(FrappeTestCase):
 
 		ev_list3 = get_events("2015-02-01", "2015-02-01", "Administrator", for_reminder=True)
 		self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list3))))
+
+	def test_quaterly_repeat(self):
+		ev = frappe.get_doc(
+			{
+				"doctype": "Event",
+				"subject": "_Test Event",
+				"starts_on": "2023-02-17",
+				"repeat_till": "2024-02-17",
+				"event_type": "Public",
+				"repeat_this_event": 1,
+				"repeat_on": "Quarterly",
+			}
+		)
+		ev.insert()
+		# Test Quaterly months
+		ev_list = get_events("2023-02-17", "2023-02-17", "Administrator", for_reminder=True)
+		self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list))))
+
+		ev_list1 = get_events("2023-05-17", "2023-05-17", "Administrator", for_reminder=True)
+		self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list1))))
+
+		ev_list2 = get_events("2023-08-17", "2023-08-17", "Administrator", for_reminder=True)
+		self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list2))))
+
+		ev_list3 = get_events("2023-11-17", "2023-11-17", "Administrator", for_reminder=True)
+		self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list3))))
+
+		# Test before event start date and after event end date
+		ev_list4 = get_events("2022-11-17", "2022-11-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
+
+		ev_list4 = get_events("2024-02-17", "2024-02-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
+
+		# Test months that aren't part of the quarterly cycle
+		ev_list4 = get_events("2023-12-17", "2023-12-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
+
+		ev_list4 = get_events("2023-03-17", "2023-03-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
+
+	def test_half_yearly_repeat(self):
+		ev = frappe.get_doc(
+			{
+				"doctype": "Event",
+				"subject": "_Test Event",
+				"starts_on": "2023-02-17",
+				"repeat_till": "2024-02-17",
+				"event_type": "Public",
+				"repeat_this_event": 1,
+				"repeat_on": "Half Yearly",
+			}
+		)
+		ev.insert()
+		# Test Half Yearly months
+		ev_list = get_events("2023-02-17", "2023-02-17", "Administrator", for_reminder=True)
+		self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list))))
+
+		ev_list1 = get_events("2023-08-17", "2023-08-17", "Administrator", for_reminder=True)
+		self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list1))))
+
+		# Test before event start date and after event end date
+		ev_list4 = get_events("2022-08-17", "2022-08-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
+
+		ev_list4 = get_events("2024-02-17", "2024-02-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
+
+		# Test months that aren't part of the half yearly cycle
+		ev_list4 = get_events("2023-12-17", "2023-12-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
+
+		ev_list4 = get_events("2023-05-17", "2023-05-17", "Administrator", for_reminder=True)
+		self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))

frappe/desk/doctype/kanban_board/kanban_board.py

@@ -238,7 +238,7 @@ def update_column_order(board_name, order):
 	new_columns = []
 
 	for col in order:
-		for column in old_columns:
+		for column in list(old_columns):
 			if col == column.column_name:
 				new_columns.append(column)
 				old_columns.remove(column)

frappe/desk/doctype/note/note.py

@@ -57,4 +57,4 @@ def get_permission_query_conditions(user):
 
 
 def has_permission(doc, user):
-	return doc.public or doc.owner == user
+	return bool(doc.public or doc.owner == user)

frappe/desk/doctype/notification_log/notification_log.py

@@ -29,6 +29,7 @@ class NotificationLog(Document):
 		read: DF.Check
 		subject: DF.Text | None
 		type: DF.Literal["Mention", "Energy Point", "Assignment", "Share", "Alert"]
+
 	# end: auto-generated types
 	def after_insert(self):
 		frappe.publish_realtime("notification", after_commit=True, user=self.for_user)
@@ -115,18 +116,17 @@ def _get_user_ids(user_emails):
 	return [user for user in user_names if is_notifications_enabled(user)]
 
 
-def send_notification_email(doc):
-
+def send_notification_email(doc: NotificationLog):
 	if doc.type == "Energy Point" and doc.email_content is None:
 		return
 
 	from frappe.utils import get_url_to_form, strip_html
 
-	email = frappe.db.get_value("User", doc.for_user, "email")
-	if not email:
+	user = frappe.db.get_value("User", doc.for_user, fieldname=["email", "language"], as_dict=True)
+	if not user:
 		return
 
-	header = get_email_header(doc)
+	header = get_email_header(doc, user.language)
 	email_subject = strip_html(doc.subject)
 	args = {
 		"body_content": doc.subject,
@@ -140,7 +140,7 @@ def send_notification_email(doc):
 		args["doc_link"] = get_url_to_form(doc.document_type, doc.document_name)
 
 	frappe.sendmail(
-		recipients=email,
+		recipients=user.email,
 		subject=email_subject,
 		template="new_notification",
 		args=args,
@@ -149,14 +149,14 @@ def send_notification_email(doc):
 	)
 
 
-def get_email_header(doc):
+def get_email_header(doc, language: str | None = None):
 	docname = doc.document_name
 	header_map = {
-		"Default": _("New Notification"),
-		"Mention": _("New Mention on {0}").format(docname),
-		"Assignment": _("Assignment Update on {0}").format(docname),
-		"Share": _("New Document Shared {0}").format(docname),
-		"Energy Point": _("Energy Point Update on {0}").format(docname),
+		"Default": _("New Notification", lang=language),
+		"Mention": _("New Mention on {0}", lang=language).format(docname),
+		"Assignment": _("Assignment Update on {0}", lang=language).format(docname),
+		"Share": _("New Document Shared {0}", lang=language).format(docname),
+		"Energy Point": _("Energy Point Update on {0}", lang=language).format(docname),
 	}
 
 	return header_map[doc.type or "Default"]

frappe/desk/doctype/tag/tag.py

@@ -109,7 +109,7 @@ class DocTags:
 			tags = ""
 		else:
 			tl = unique(filter(lambda x: x, tl))
-			tags = "," + ",".join(tl)
+			tags = ",".join(tl)
 		try:
 			frappe.db.sql(
 				"update `tab{}` set _user_tags={} where name={}".format(self.dt, "%s", "%s"), (tags, dn)

frappe/desk/doctype/workspace/workspace.py

@@ -11,6 +11,7 @@ from frappe.desk.utils import validate_route_conflict
 from frappe.model.document import Document
 from frappe.model.rename_doc import rename_doc
 from frappe.modules.export_file import delete_folder, export_to_files
+from frappe.utils import strip_html
 
 
 class Workspace(Document):
@@ -65,6 +66,8 @@ class Workspace(Document):
 		title: DF.Data
 	# end: auto-generated types
 	def validate(self):
+		self.title = strip_html(self.title)
+
 		if self.public and not is_workspace_manager() and not disable_saving_as_public():
 			frappe.throw(_("You need to be Workspace Manager to edit this document"))
 		if self.has_value_changed("title"):
@@ -183,6 +186,7 @@ class Workspace(Document):
 					"label": card.get("label"),
 					"type": "Card Break",
 					"icon": card.get("icon"),
+					"description": card.get("description"),
 					"hidden": card.get("hidden") or False,
 					"link_count": card.get("link_count"),
 					"idx": 1 if not self.links else self.links[-1].idx + 1,
@@ -275,6 +279,8 @@ def save_page(title, public, new_widgets, blocks):
 	pages = frappe.get_all("Workspace", filters=filters)
 	if pages:
 		doc = frappe.get_doc("Workspace", pages[0])
+	else:
+		frappe.throw(_("Workspace not found"), frappe.DoesNotExistError)
 
 	doc.content = blocks
 

frappe/desk/doctype/workspace_link/workspace_link.json

@@ -8,6 +8,7 @@
   "type",
   "label",
   "icon",
+  "description",
   "hidden",
   "link_details_section",
   "link_type",
@@ -107,12 +108,20 @@
    "fieldtype": "Int",
    "hidden": 1,
    "label": "Link Count"
+  },
+  {
+   "depends_on": "eval:doc.type == \"Card Break\"",
+   "fieldname": "description",
+   "fieldtype": "HTML Editor",
+   "ignore_xss_filter": 1,
+   "label": "Description",
+   "max_height": "7rem"
   }
  ],
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-06-01 11:23:28.990593",
+ "modified": "2024-01-23 17:39:16.833318",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace Link",
@@ -121,5 +130,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/desk/doctype/workspace_link/workspace_link.py

@@ -15,6 +15,7 @@ class WorkspaceLink(Document):
 		from frappe.types import DF
 
 		dependencies: DF.Data | None
+		description: DF.HTMLEditor | None
 		hidden: DF.Check
 		icon: DF.Data | None
 		is_query_report: DF.Check
@@ -29,4 +30,5 @@ class WorkspaceLink(Document):
 		parenttype: DF.Data
 		type: DF.Literal["Link", "Card Break"]
 	# end: auto-generated types
+
 	pass

frappe/desk/form/assign_to.py

@@ -175,6 +175,20 @@ def remove(doctype, name, assign_to, ignore_permissions=False):
 	)
 
 
+@frappe.whitelist()
+def remove_multiple(doctype, names, ignore_permissions=False):
+	docname_list = json.loads(names)
+
+	for name in docname_list:
+		assignments = get({"doctype": doctype, "name": name})
+
+		if not assignments:
+			continue
+
+		for assignment in assignments:
+			remove(doctype, name, assignment.get("owner"), ignore_permissions)
+
+
 @frappe.whitelist()
 def close(doctype: str, name: str, assign_to: str, ignore_permissions=False):
 	if assign_to != frappe.session.user:
@@ -253,8 +267,10 @@ def notify_assignment(
 	if not (assigned_by and allocated_to and doc_type and doc_name):
 		return
 
+	assigned_user = frappe.db.get_value("User", allocated_to, ["language", "enabled"], as_dict=True)
+
 	# return if self assigned or user disabled
-	if assigned_by == allocated_to or not frappe.db.get_value("User", allocated_to, "enabled"):
+	if assigned_by == allocated_to or not assigned_user.enabled:
 		return
 
 	# Search for email address in description -- i.e. assignee
@@ -263,14 +279,16 @@ def notify_assignment(
 	description_html = f"<div>{description}</div>" if description else None
 
 	if action == "CLOSE":
-		subject = _("Your assignment on {0} {1} has been removed by {2}").format(
-			frappe.bold(_(doc_type)), get_title_html(title), frappe.bold(user_name)
-		)
+		subject = _(
+			"Your assignment on {0} {1} has been removed by {2}", lang=assigned_user.language
+		).format(frappe.bold(_(doc_type)), get_title_html(title), frappe.bold(user_name))
 	else:
 		user_name = frappe.bold(user_name)
-		document_type = frappe.bold(_(doc_type))
+		document_type = frappe.bold(_(doc_type, lang=assigned_user.language))
 		title = get_title_html(title)
-		subject = _("{0} assigned a new task {1} {2} to you").format(user_name, document_type, title)
+		subject = _("{0} assigned a new task {1} {2} to you", lang=assigned_user.language).format(
+			user_name, document_type, title
+		)
 
 	notification_doc = {
 		"type": "Assignment",

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -476,18 +476,15 @@ frappe.setup.slides_settings = [
 
 		onload: function (slide) {
 			if (frappe.session.user !== "Administrator") {
-				slide.form.fields_dict.email.$wrapper.toggle(false);
-				slide.form.fields_dict.password.$wrapper.toggle(false);
-
-				// remove password field
-				delete slide.form.fields_dict.password;
-
-				if (frappe.boot.user.first_name || frappe.boot.user.last_name) {
+				const { first_name, last_name, email } = frappe.boot.user;
+				if (first_name || last_name) {
 					slide.form.fields_dict.full_name.set_input(
-						[frappe.boot.user.first_name, frappe.boot.user.last_name].join(" ").trim()
+						[first_name, last_name].join(" ").trim()
 					);
 				}
-				delete slide.form.fields_dict.email;
+				slide.form.fields_dict.email.set_input(email);
+				slide.form.fields_dict.email.df.read_only = 1;
+				slide.form.fields_dict.email.refresh();
 			} else {
 				slide.form.fields_dict.email.df.reqd = 1;
 				slide.form.fields_dict.email.refresh();

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -7,14 +7,14 @@ import frappe
 from frappe import _
 from frappe.geo.country_info import get_country_info
 from frappe.permissions import AUTOMATIC_ROLES
-from frappe.translate import get_messages_for_boot, send_translations, set_default_language
+from frappe.translate import send_translations, set_default_language
 from frappe.utils import cint, now, strip
 from frappe.utils.password import update_password
 
 from . import install_fixtures
 
 
-def get_setup_stages(args):
+def get_setup_stages(args):  # nosemgrep
 
 	# App setup stage functions should not include frappe.db.commit
 	# That is done by frappe after successful completion of all stages
@@ -104,18 +104,18 @@ def process_setup_stages(stages, user_input, is_background_task=False):
 		frappe.flags.in_setup_wizard = False
 
 
-def update_global_settings(args):
+def update_global_settings(args):  # nosemgrep
 	if args.language and args.language != "English":
 		set_default_language(get_language_code(args.lang))
 		frappe.db.commit()
 	frappe.clear_cache()
 
 	update_system_settings(args)
-	update_user_name(args)
+	create_or_update_user(args)
 	set_timezone(args)
 
 
-def run_post_setup_complete(args):
+def run_post_setup_complete(args):  # nosemgrep
 	disable_future_access()
 	frappe.db.commit()
 	frappe.clear_cache()
@@ -124,20 +124,20 @@ def run_post_setup_complete(args):
 	frappe.get_cached_doc("System Settings") and frappe.get_doc("System Settings")
 
 
-def run_setup_success(args):
+def run_setup_success(args):  # nosemgrep
 	for hook in frappe.get_hooks("setup_wizard_success"):
 		frappe.get_attr(hook)(args)
 	install_fixtures.install()
 
 
-def get_stages_hooks(args):
+def get_stages_hooks(args):  # nosemgrep
 	stages = []
 	for method in frappe.get_hooks("setup_wizard_stages"):
 		stages += frappe.get_attr(method)(args)
 	return stages
 
 
-def get_setup_complete_hooks(args):
+def get_setup_complete_hooks(args):  # nosemgrep
 	return [
 		{
 			"status": "Executing method",
@@ -154,7 +154,7 @@ def get_setup_complete_hooks(args):
 	]
 
 
-def handle_setup_exception(args):
+def handle_setup_exception(args):  # nosemgrep
 	frappe.db.rollback()
 	if args:
 		traceback = frappe.get_traceback(with_context=True)
@@ -163,7 +163,7 @@ def handle_setup_exception(args):
 			frappe.get_attr(hook)(traceback, args)
 
 
-def update_system_settings(args):
+def update_system_settings(args):  # nosemgrep
 	number_format = get_country_info(args.get("country")).get("number_format", "#,###.##")
 
 	# replace these as float number formats, as they have 0 precision
@@ -194,72 +194,51 @@ def update_system_settings(args):
 		frappe.db.set_default("session_recording_start", now())
 
 
-def update_user_name(args):
+def create_or_update_user(args):  # nosemgrep
+	email = args.get("email")
+	if not email:
+		return
+
 	first_name, last_name = args.get("full_name", ""), ""
 	if " " in first_name:
 		first_name, last_name = first_name.split(" ", 1)
 
-	if args.get("email"):
-		if frappe.db.exists("User", args.get("email")):
-			# running again
-			return
-
-		args["name"] = args.get("email")
-
+	if user := frappe.db.get_value("User", email, ["first_name", "last_name"], as_dict=True):
+		if user.first_name != first_name or user.last_name != last_name:
+			(
+				frappe.qb.update("User")
+				.set("first_name", first_name)
+				.set("last_name", last_name)
+				.set("full_name", args.get("full_name"))
+			).run()
+	else:
 		_mute_emails, frappe.flags.mute_emails = frappe.flags.mute_emails, True
-		doc = frappe.get_doc(
+
+		user = frappe.new_doc("User")
+		user.update(
 			{
-				"doctype": "User",
-				"email": args.get("email"),
+				"email": email,
 				"first_name": first_name,
 				"last_name": last_name,
 			}
 		)
+		user.append_roles(*_get_default_roles())
+		user.flags.no_welcome_mail = True
+		user.insert()
 
-		doc.append_roles(*_get_default_roles())
-		doc.flags.no_welcome_mail = True
-		doc.insert()
 		frappe.flags.mute_emails = _mute_emails
-		update_password(args.get("email"), args.get("password"))
 
-	elif first_name:
-		args.update({"name": frappe.session.user, "first_name": first_name, "last_name": last_name})
-
-		frappe.db.sql(
-			"""update `tabUser` SET first_name=%(first_name)s,
-			last_name=%(last_name)s WHERE name=%(name)s""",
-			args,
-		)
-
-	if args.get("attach_user"):
-		attach_user = args.get("attach_user").split(",")
-		if len(attach_user) == 3:
-			filename, filetype, content = attach_user
-			_file = frappe.get_doc(
-				{
-					"doctype": "File",
-					"file_name": filename,
-					"attached_to_doctype": "User",
-					"attached_to_name": args.get("name"),
-					"content": content,
-					"decode": True,
-				}
-			)
-			_file.save()
-			fileurl = _file.file_url
-			frappe.db.set_value("User", args.get("name"), "user_image", fileurl)
-
-	if args.get("name"):
-		add_all_roles_to(args.get("name"))
+	if args.get("password"):
+		update_password(email, args.get("password"))
 
 
-def set_timezone(args):
+def set_timezone(args):  # nosemgrep
 	if args.get("timezone"):
 		for name in frappe.STANDARD_USERS:
 			frappe.db.set_value("User", name, "time_zone", args.get("timezone"))
 
 
-def parse_args(args):
+def parse_args(args):  # nosemgrep
 	if not args:
 		args = frappe.local.form_dict
 	if isinstance(args, str):
@@ -304,6 +283,8 @@ def disable_future_access():
 def load_messages(language):
 	"""Load translation messages for given language from all `setup_wizard_requires`
 	javascript files"""
+	from frappe.translate import get_messages_for_boot
+
 	frappe.clear_cache()
 	set_default_language(get_language_code(language))
 	frappe.db.commit()
@@ -342,7 +323,7 @@ def load_user_details():
 	}
 
 
-def prettify_args(args):
+def prettify_args(args):  # nosemgrep
 	# remove attachments
 	for key, val in args.items():
 		if isinstance(val, str) and "data:image" in val:
@@ -355,7 +336,7 @@ def prettify_args(args):
 	return pretty_args
 
 
-def email_setup_wizard_exception(traceback, args):
+def email_setup_wizard_exception(traceback, args):  # nosemgrep
 	if not frappe.conf.setup_wizard_exception_email:
 		return
 
@@ -400,7 +381,7 @@ def email_setup_wizard_exception(traceback, args):
 	)
 
 
-def log_setup_wizard_exception(traceback, args):
+def log_setup_wizard_exception(traceback, args):  # nosemgrep
 	with open("../logs/setup-wizard.log", "w+") as setup_log:
 		setup_log.write(traceback)
 		setup_log.write(json.dumps(args))

frappe/email/doctype/email_account/email_account.py

@@ -498,6 +498,8 @@ class EmailAccount(Document):
 			self.set_failed_attempts_count(self.get_failed_attempts_count() + 1)
 
 	def _disable_broken_incoming_account(self, description):
+		if frappe.flags.in_test:
+			return
 		self.db_set("enable_incoming", 0)
 
 		for user in get_system_managers(only_name=True):

frappe/email/doctype/email_account/test_email_account.py

@@ -13,12 +13,8 @@ from frappe.desk.form.load import get_attachments
 from frappe.email.doctype.email_account.email_account import notify_unreplied
 from frappe.email.email_body import get_message_id
 from frappe.email.receive import Email, InboundMail, SentEmailInInboxError
-from frappe.test_runner import make_test_records
 from frappe.tests.utils import FrappeTestCase
 
-make_test_records("User")
-make_test_records("Email Account")
-
 
 class TestEmailAccount(FrappeTestCase):
 	@classmethod
@@ -65,9 +61,18 @@ class TestEmailAccount(FrappeTestCase):
 		self.assertTrue(frappe.db.get_value(comm.reference_doctype, comm.reference_name, "name"))
 
 	def test_unread_notification(self):
-		self.test_incoming()
+		todo = frappe.get_last_doc("ToDo")
 
-		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
+		comm = frappe.new_doc(
+			"Communication",
+			sender="test_sender@example.com",
+			subject="test unread reminder",
+			sent_or_received="Received",
+			reference_doctype=todo.doctype,
+			reference_name=todo.name,
+			email_account="_Test Email Account 1",
+		)
+		comm.insert()
 		comm.db_set("creation", datetime.now() - timedelta(seconds=30 * 60))
 
 		frappe.db.delete("Email Queue")
@@ -78,7 +83,6 @@ class TestEmailAccount(FrappeTestCase):
 				{
 					"reference_doctype": comm.reference_doctype,
 					"reference_name": comm.reference_name,
-					"status": "Not Sent",
 				},
 			)
 		)

frappe/email/doctype/email_queue/email_queue.py

@@ -6,7 +6,7 @@ import quopri
 import traceback
 from contextlib import suppress
 from email.parser import Parser
-from email.policy import SMTPUTF8, default
+from email.policy import SMTP
 
 import frappe
 from frappe import _, safe_encode, task
@@ -169,7 +169,9 @@ class EmailQueue(Document):
 				else:
 					if not frappe.flags.in_test or frappe.flags.testing_email:
 						ctx.smtp_server.session.sendmail(
-							from_addr=self.sender, to_addrs=recipient.recipient, msg=message
+							from_addr=self.sender,
+							to_addrs=recipient.recipient,
+							msg=message.decode("utf-8").encode(),
 						)
 
 				ctx.update_recipient_status_to_sent(recipient)
@@ -264,7 +266,7 @@ class SendMailContext:
 	@savepoint(catch=Exception)
 	def notify_failed_email(self):
 		# Parse the email body to extract the subject
-		subject = Parser(policy=default).parsestr(self.queue_doc.message)["Subject"]
+		subject = Parser(policy=SMTP).parsestr(self.queue_doc.message)["Subject"]
 
 		# Construct the notification
 		notification = frappe.new_doc("Notification Log")
@@ -281,7 +283,7 @@ class SendMailContext:
 		recipient.update_db(status="Sent", commit=True)
 
 	def get_message_object(self, message):
-		return Parser(policy=SMTPUTF8).parsestr(message)
+		return Parser(policy=SMTP).parsestr(message)
 
 	def message_placeholder(self, placeholder_key):
 		# sourcery skip: avoid-builtin-shadow
@@ -293,9 +295,10 @@ class SendMailContext:
 		}
 		return map.get(placeholder_key)
 
-	def build_message(self, recipient_email):
+	def build_message(self, recipient_email) -> bytes:
 		"""Build message specific to the recipient."""
 		message = self.queue_doc.message
+
 		if not message:
 			return ""
 

frappe/email/doctype/notification/notification.py

@@ -366,7 +366,9 @@ def get_context(context):
 
 			# For sending messages to specified role
 			if recipient.receiver_by_role:
-				receiver_list += get_info_based_on_role(recipient.receiver_by_role, "mobile_no")
+				receiver_list += get_info_based_on_role(
+					recipient.receiver_by_role, "mobile_no", ignore_permissions=True
+				)
 
 		return receiver_list
 

frappe/email/email_body.py

@@ -1,6 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-
 import email.utils
 import os
 import re
@@ -136,8 +135,8 @@ class EMail:
 		self.subject = subject
 		self.expose_recipients = expose_recipients
 
-		self.msg_root = MIMEMultipart("mixed", policy=policy.SMTPUTF8)
-		self.msg_alternative = MIMEMultipart("alternative", policy=policy.SMTPUTF8)
+		self.msg_root = MIMEMultipart("mixed", policy=policy.SMTP)
+		self.msg_alternative = MIMEMultipart("alternative", policy=policy.SMTP)
 		self.msg_root.attach(self.msg_alternative)
 		self.cc = cc or []
 		self.bcc = bcc or []
@@ -186,7 +185,7 @@ class EMail:
 		"""
 		from email.mime.text import MIMEText
 
-		part = MIMEText(message, "plain", "utf-8", policy=policy.SMTPUTF8)
+		part = MIMEText(message, "plain", "utf-8", policy=policy.SMTP)
 		self.msg_alternative.attach(part)
 
 	def set_part_html(self, message, inline_images):
@@ -199,9 +198,9 @@ class EMail:
 			message, _inline_images = replace_filename_with_cid(message)
 
 			# prepare parts
-			msg_related = MIMEMultipart("related", policy=policy.SMTPUTF8)
+			msg_related = MIMEMultipart("related", policy=policy.SMTP)
 
-			html_part = MIMEText(message, "html", "utf-8", policy=policy.SMTPUTF8)
+			html_part = MIMEText(message, "html", "utf-8", policy=policy.SMTP)
 			msg_related.attach(html_part)
 
 			for image in _inline_images:
@@ -215,7 +214,7 @@ class EMail:
 
 			self.msg_alternative.attach(msg_related)
 		else:
-			self.msg_alternative.attach(MIMEText(message, "html", "utf-8", policy=policy.SMTPUTF8))
+			self.msg_alternative.attach(MIMEText(message, "html", "utf-8", policy=policy.SMTP))
 
 	def set_html_as_text(self, html):
 		"""Set plain text from HTML"""
@@ -228,7 +227,7 @@ class EMail:
 		from email.mime.text import MIMEText
 
 		maintype, subtype = mime_type.split("/")
-		part = MIMEText(message, _subtype=subtype, policy=policy.SMTPUTF8)
+		part = MIMEText(message, _subtype=subtype, policy=policy.SMTP)
 
 		if as_attachment:
 			part.add_header("Content-Disposition", "attachment", filename=filename)
@@ -342,7 +341,7 @@ class EMail:
 		"""validate, build message and convert to string"""
 		self.validate()
 		self.make()
-		return self.msg_root.as_string(policy=policy.SMTPUTF8)
+		return self.msg_root.as_string(policy=policy.SMTP)
 
 
 def get_formatted_html(

frappe/geo/doctype/currency/currency.json

@@ -82,7 +82,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-07-04 09:42:52.425440",
+ "modified": "2024-01-17 15:37:31.605278",
  "modified_by": "Administrator",
  "module": "Geo",
  "name": "Currency",
@@ -102,6 +102,10 @@
    "share": 1,
    "write": 1
   },
+  {
+   "read": 1,
+   "role": "Accounts Manager"
+  },
   {
    "read": 1,
    "role": "Accounts User"

frappe/gettext/extractors/README.md

@@ -0,0 +1,2 @@
+Extractors should run on source files only.
+They should not depend on an acitive web server or database connection.

frappe/gettext/extractors/doctype.py

@@ -0,0 +1,72 @@
+import json
+
+
+def extract(fileobj, *args, **kwargs):
+	"""
+	Extract messages from DocType JSON files. To be used to babel extractor
+	:param fileobj: the file-like object the messages should be extracted from
+	:rtype: `iterator`
+	"""
+	data = json.load(fileobj)
+
+	if isinstance(data, list):
+		return
+
+	doctype = data.get("name")
+
+	yield None, "_", doctype, ["Name of a DocType"]
+
+	messages = []
+	fields = data.get("fields", [])
+	links = data.get("links", [])
+
+	for field in fields:
+		fieldtype = field.get("fieldtype")
+		label = field.get("label")
+
+		if label:
+			messages.append((label, f"Label of a {fieldtype} field in DocType '{doctype}'"))
+			_label = label
+		else:
+			_label = field.get("fieldname")
+
+		if description := field.get("description"):
+			messages.append(
+				(description, f"Description of the '{_label}' ({fieldtype}) field in DocType '{doctype}'")
+			)
+
+		if message := field.get("options"):
+			if fieldtype == "Select":
+				select_options = [option for option in message.split("\n") if option and not option.isdigit()]
+
+				if select_options and "icon" in select_options[0]:
+					continue
+
+				messages.extend(
+					(
+						option,
+						f"Option for the '{_label}' ({fieldtype}) field in DocType '{doctype}'",
+					)
+					for option in select_options
+				)
+			elif fieldtype == "HTML":
+				messages.append(
+					(message, f"Content of the '{_label}' ({fieldtype}) field in DocType '{doctype}'")
+				)
+
+	for link in links:
+		if group := link.get("group"):
+			messages.append((group, f"Group in {doctype}'s connections"))
+
+		if link_doctype := link.get("link_doctype"):
+			messages.append((link_doctype, f"Linked DocType in {doctype}'s connections"))
+
+	# By using "pgettext" as the function name we can supply the doctype as context
+	yield from ((None, "pgettext", (doctype, message), [comment]) for message, comment in messages)
+
+	# Role names do not get context because they are used with multiple doctypes
+	yield from (
+		(None, "_", perm["role"], ["Name of a role"])
+		for perm in data.get("permissions", [])
+		if "role" in perm
+	)

frappe/gettext/extractors/javascript.py

@@ -0,0 +1,163 @@
+from io import BufferedReader
+
+
+def extract(fileobj: BufferedReader, keywords: str, comment_tags: tuple, options: dict):
+	code = fileobj.read().decode("utf-8")
+
+	for lineno, funcname, messages in extract_javascript(code, "__", options):
+		if not messages or not messages[0]:
+			continue
+
+		# `funcname` here will be `__` which is our translation function. We
+		# have to convert it back to usual function names
+		funcname = "gettext"
+
+		if isinstance(messages, tuple):
+			if len(messages) == 3 and messages[2]:
+				funcname = "pgettext"
+				messages = (messages[2], messages[0])
+			else:
+				messages = messages[0]
+
+		yield lineno, funcname, messages, []
+
+
+def extract_javascript(code, keywords=("__",), options=None):
+	"""Extract messages from JavaScript source code.
+
+	This is a modified version of babel's JS parser. Reused under BSD license.
+	License: https://github.com/python-babel/babel/blob/master/LICENSE
+
+	Changes from upstream:
+	- Preserve arguments, babel's parser flattened all values in args,
+	  we need order because we use different syntax for translation
+	  which can contain 2nd arg which is array of many values. If
+	  argument is non-primitive type then value is NOT returned in
+	  args.
+	  E.g. __("0", ["1", "2"], "3") -> ("0", None, "3")
+	- remove comments support
+	- changed signature to accept string directly.
+
+	:param code: code as string
+	:param keywords: a list of keywords (i.e. function names) that should be
+	                 recognized as translation functions
+	:param options: a dictionary of additional options (optional)
+	                Supported options are:
+	                * `template_string` -- set to false to disable ES6
+	                                       template string support.
+	"""
+	from babel.messages.jslexer import Token, tokenize, unquote_string
+
+	if options is None:
+		options = {}
+
+	funcname = message_lineno = None
+	messages = []
+	last_argument = None
+	concatenate_next = False
+	last_token = None
+	call_stack = -1
+
+	# Tree level = depth inside function call tree
+	#  Example: __("0", ["1", "2"], "3")
+	# Depth         __()
+	#             /   |   \
+	#   0       "0" [...] "3"  <- only 0th level strings matter
+	#                /  \
+	#   1          "1"  "2"
+	tree_level = 0
+	opening_operators = {"[", "{"}
+	closing_operators = {"]", "}"}
+	all_container_operators = opening_operators.union(closing_operators)
+	dotted = any("." in kw for kw in keywords)
+
+	for token in tokenize(
+		code,
+		jsx=True,
+		template_string=options.get("template_string", True),
+		dotted=dotted,
+	):
+		if (  # Turn keyword`foo` expressions into keyword("foo") calls:
+			funcname
+			and (last_token and last_token.type == "name")  # have a keyword...
+			and token.type  # we've seen nothing after the keyword...
+			== "template_string"  # this is a template string
+		):
+			message_lineno = token.lineno
+			messages = [unquote_string(token.value)]
+			call_stack = 0
+			tree_level = 0
+			token = Token("operator", ")", token.lineno)
+
+		if token.type == "operator" and token.value == "(":
+			if funcname:
+				message_lineno = token.lineno
+				call_stack += 1
+
+		elif call_stack >= 0 and token.type == "operator" and token.value in all_container_operators:
+			if token.value in opening_operators:
+				tree_level += 1
+			if token.value in closing_operators:
+				tree_level -= 1
+
+		elif call_stack == -1 and token.type == "linecomment" or token.type == "multilinecomment":
+			pass  # ignore comments
+
+		elif funcname and call_stack == 0:
+			if token.type == "operator" and token.value == ")":
+				if last_argument is not None:
+					messages.append(last_argument)
+				if len(messages) > 1:
+					messages = tuple(messages)
+				elif messages:
+					messages = messages[0]
+				else:
+					messages = None
+
+				if messages is not None:
+					yield (message_lineno, funcname, messages)
+
+				funcname = message_lineno = last_argument = None
+				concatenate_next = False
+				messages = []
+				call_stack = -1
+				tree_level = 0
+
+			elif token.type in ("string", "template_string"):
+				new_value = unquote_string(token.value)
+				if tree_level > 0:
+					pass
+				elif concatenate_next:
+					last_argument = (last_argument or "") + new_value
+					concatenate_next = False
+				else:
+					last_argument = new_value
+
+			elif token.type == "operator":
+				if token.value == ",":
+					if last_argument is not None:
+						messages.append(last_argument)
+						last_argument = None
+					else:
+						if tree_level == 0:
+							messages.append(None)
+					concatenate_next = False
+				elif token.value == "+":
+					concatenate_next = True
+
+		elif call_stack > 0 and token.type == "operator" and token.value == ")":
+			call_stack -= 1
+			tree_level = 0
+
+		elif funcname and call_stack == -1:
+			funcname = None
+
+		elif (
+			call_stack == -1
+			and token.type == "name"
+			and token.value in keywords
+			and (last_token is None or last_token.type != "name" or last_token.value != "function")
+		):
+			funcname = token.value
+
+		last_token = token

frappe/gettext/extractors/jinja2.py

@@ -0,0 +1,11 @@
+from jinja2.ext import babel_extract
+
+
+def extract(*args, **kwargs):
+	"""Reuse the babel_extract function from jinja2.ext, but handle our own implementation of `_()`"""
+	for lineno, funcname, messages, comments in babel_extract(*args, **kwargs):
+		if funcname == "_" and isinstance(messages, tuple) and len(messages) > 1:
+			funcname = "pgettext"
+			messages = (messages[-1], messages[0])  # (context, message)
+
+		yield lineno, funcname, messages, comments

frappe/gettext/extractors/module_onboarding.py

@@ -0,0 +1,30 @@
+import json
+
+
+def extract(fileobj, *args, **kwargs):
+	"""
+	Extract messages from Module Onboarding JSON files.
+
+	:param fileobj: the file-like object the messages should be extracted from
+	:rtype: `iterator`
+	"""
+	data = json.load(fileobj)
+
+	if isinstance(data, list):
+		return
+
+	if data.get("doctype") != "Module Onboarding":
+		return
+
+	onboarding_name = data.get("name")
+
+	if title := data.get("title"):
+		yield None, "_", title, [f"Title of the Module Onboarding '{onboarding_name}'"]
+
+	if subtitle := data.get("subtitle"):
+		yield None, "_", subtitle, [f"Subtitle of the Module Onboarding '{onboarding_name}'"]
+
+	if success_message := data.get("success_message"):
+		yield None, "_", success_message, [
+			f"Success message of the Module Onboarding '{onboarding_name}'"
+		]