From 7e6e7aa0556953d0a5a88a3304e83f71634f4ad7 Mon Sep 17 00:00:00 2001
From: Devin Slauenwhite <devin.slauenwhite@gmail.com>
Date: Fri, 13 Jan 2023 13:53:13 -0500
Subject: [PATCH 01/60] fix: ambiguous linked tables

---
 frappe/model/db_query.py | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 1d156d0d1a..2718dd1180 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -5,6 +5,7 @@
 import copy
 import json
 import re
+from collections import Counter
 from datetime import datetime
 
 import frappe
@@ -57,6 +58,7 @@ class DatabaseQuery:
 		self.doctype = doctype
 		self.tables = []
 		self.link_tables = []
+		self.linked_table_counter = Counter()
 		self.conditions = []
 		self.or_conditions = []
 		self.fields = None
@@ -242,7 +244,7 @@ class DatabaseQuery:
 
 		# left join link tables
 		for link in self.link_tables:
-			args.tables += f" {self.join} `tab{link.doctype}` on (`tab{link.doctype}`.`name` = {self.tables[0]}.`{link.fieldname}`)"
+			args.tables += f" {self.join} {link.table_name} {link.table_alias} on ({link.table_alias}.`name` = {self.tables[0]}.`{link.fieldname}`)"
 
 		if self.grouped_or_conditions:
 			self.conditions.append(f"({' or '.join(self.grouped_or_conditions)})")
@@ -326,8 +328,10 @@ class DatabaseQuery:
 				linked_field = frappe.get_meta(self.doctype).get_field(linked_fieldname)
 				linked_doctype = linked_field.options
 				if linked_field.fieldtype == "Link":
-					self.append_link_table(linked_doctype, linked_fieldname)
-				field = f"`tab{linked_doctype}`.`{fieldname}`"
+					linked_table = self.append_link_table(linked_doctype, linked_fieldname)
+					field = f"{linked_table.table_alias}.`{fieldname}`"
+				else:
+					field = f"`tab{linked_doctype}`.`{fieldname}`"
 				if alias:
 					field = f"{field} as {alias}"
 				self.fields[self.fields.index(original_field)] = field
@@ -437,6 +441,12 @@ class DatabaseQuery:
 
 				table_name = field.split(".")[0]
 
+				# Check if table_name is a linked_table alias
+				for linked_table in self.link_tables:
+					if linked_table.table_alias == table_name:
+						table_name = linked_table.table_name
+						break
+
 				if table_name.lower().startswith("group_concat("):
 					table_name = table_name[13:]
 				if not table_name[0] == "`":
@@ -452,14 +462,20 @@ class DatabaseQuery:
 		self.check_read_permission(doctype)
 
 	def append_link_table(self, doctype, fieldname):
-		for d in self.link_tables:
-			if d.doctype == doctype and d.fieldname == fieldname:
-				return
+		for linked_table in self.link_tables:
+			if linked_table.doctype == doctype and linked_table.fieldname == fieldname:
+				return linked_table
 
 		self.check_read_permission(doctype)
-		self.link_tables.append(
-			frappe._dict(doctype=doctype, fieldname=fieldname, table_name=f"`tab{doctype}`")
+		self.linked_table_counter.update((doctype,))
+		linked_table = frappe._dict(
+			doctype=doctype,
+			fieldname=fieldname,
+			table_name=f"`tab{doctype}`",
+			table_alias=f"`tab{doctype}_{self.linked_table_counter[doctype]}`",
 		)
+		self.link_tables.append(linked_table)
+		return linked_table
 
 	def check_read_permission(self, doctype):
 		if not self.flags.ignore_permissions and not frappe.has_permission(

From af579e3192266e643b2a2419d55752a7d28e4551 Mon Sep 17 00:00:00 2001
From: Devin Slauenwhite <devin.slauenwhite@gmail.com>
Date: Fri, 13 Jan 2023 14:09:09 -0500
Subject: [PATCH 02/60] test: ambiguous linked tables

---
 frappe/tests/test_db_query.py | 57 +++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py
index 1964792ea8..046f9a5dd5 100644
--- a/frappe/tests/test_db_query.py
+++ b/frappe/tests/test_db_query.py
@@ -917,6 +917,63 @@ class TestReportview(FrappeTestCase):
 		self.assertIn("ifnull", frappe.get_all("User", {"name": ("not in", [])}, run=0))
 		self.assertIn("ifnull", frappe.get_all("User", {"name": ("not in", [""])}, run=0))
 
+	def test_ambiguous_linked_tables(self):
+		from frappe.desk.reportview import get_list
+
+		frappe.get_doc(
+			{
+				"doctype": "DocType",
+				"custom": 1,
+				"module": "Custom",
+				"name": "Related Todos",
+				"naming_rule": "Random",
+				"autoname": "hash",
+				"fields": [
+					{
+						"label": "Todo One",
+						"fieldname": "todo_one",
+						"fieldtype": "Link",
+						"options": "ToDo",
+						"reqd": 1,
+					},
+					{
+						"label": "Todo Two",
+						"fieldname": "todo_two",
+						"fieldtype": "Link",
+						"options": "ToDo",
+						"reqd": 1,
+					},
+				],
+			}
+		).insert()
+
+		todo_one = frappe.get_doc(
+			{
+				"doctype": "ToDo",
+				"description": "Todo One",
+			}
+		).insert()
+
+		todo_two = frappe.get_doc(
+			{
+				"doctype": "ToDo",
+				"description": "Todo Two",
+			}
+		).insert()
+
+		frappe.get_doc(
+			{
+				"doctype": "Related Todos",
+				"todo_one": todo_one.name,
+				"todo_two": todo_two.name,
+			}
+		).insert()
+
+		frappe.form_dict.doctype = "Related Todos"
+		data = get_list()
+		frappe.form_dict.doctype = ""
+		self.assertEqual(len(data), 1)
+
 
 def add_child_table_to_blog_post():
 	child_table = frappe.get_doc(

From 1fa9c1524022b7068ada03ecb65e7649cd43fee6 Mon Sep 17 00:00:00 2001
From: Devin Slauenwhite <devin.slauenwhite@gmail.com>
Date: Fri, 13 Jan 2023 14:39:21 -0500
Subject: [PATCH 03/60] fix(test): test no pymysql.err.OperationalError

---
 frappe/tests/test_db_query.py | 72 ++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 30 deletions(-)

diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py
index 046f9a5dd5..b231a83b0e 100644
--- a/frappe/tests/test_db_query.py
+++ b/frappe/tests/test_db_query.py
@@ -918,34 +918,37 @@ class TestReportview(FrappeTestCase):
 		self.assertIn("ifnull", frappe.get_all("User", {"name": ("not in", [""])}, run=0))
 
 	def test_ambiguous_linked_tables(self):
-		from frappe.desk.reportview import get_list
+		from frappe.desk.reportview import get
 
-		frappe.get_doc(
-			{
-				"doctype": "DocType",
-				"custom": 1,
-				"module": "Custom",
-				"name": "Related Todos",
-				"naming_rule": "Random",
-				"autoname": "hash",
-				"fields": [
-					{
-						"label": "Todo One",
-						"fieldname": "todo_one",
-						"fieldtype": "Link",
-						"options": "ToDo",
-						"reqd": 1,
-					},
-					{
-						"label": "Todo Two",
-						"fieldname": "todo_two",
-						"fieldtype": "Link",
-						"options": "ToDo",
-						"reqd": 1,
-					},
-				],
-			}
-		).insert()
+		if not frappe.db.exists("DocType", "Related Todos"):
+			frappe.get_doc(
+				{
+					"doctype": "DocType",
+					"custom": 1,
+					"module": "Custom",
+					"name": "Related Todos",
+					"naming_rule": "Random",
+					"autoname": "hash",
+					"fields": [
+						{
+							"label": "Todo One",
+							"fieldname": "todo_one",
+							"fieldtype": "Link",
+							"options": "ToDo",
+							"reqd": 1,
+						},
+						{
+							"label": "Todo Two",
+							"fieldname": "todo_two",
+							"fieldtype": "Link",
+							"options": "ToDo",
+							"reqd": 1,
+						},
+					],
+				}
+			).insert()
+		else:
+			frappe.db.delete("Related Todos")
 
 		todo_one = frappe.get_doc(
 			{
@@ -970,9 +973,18 @@ class TestReportview(FrappeTestCase):
 		).insert()
 
 		frappe.form_dict.doctype = "Related Todos"
-		data = get_list()
-		frappe.form_dict.doctype = ""
-		self.assertEqual(len(data), 1)
+		frappe.form_dict.fields = [
+			"`tabRelated Todos`.`name`",
+			"`tabRelated Todos`.`todo_one`",
+			"`tabRelated Todos`.`todo_two`",
+			# because ToDo.show_title_as_field_link = 1
+			"todo_one.description as todo_one_description",
+			"todo_two.description as todo_two_description",
+		]
+
+		# Shouldn't raise pymysql.err.OperationalError: (1066, "Not unique table/alias: 'tabToDo'")
+		data = get()
+		self.assertEqual(len(data["values"]), 1)
 
 
 def add_child_table_to_blog_post():

From 943a49aabce8f17879a637a565e80ea9045a6099 Mon Sep 17 00:00:00 2001
From: Devin Slauenwhite <devin.slauenwhite@gmail.com>
Date: Fri, 3 Mar 2023 11:28:36 -0500
Subject: [PATCH 04/60] fix: perm check on alias table name

---
 frappe/model/db_query.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 4143f05630..4821f6558e 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -62,6 +62,7 @@ class DatabaseQuery:
 		self.doctype = doctype
 		self.tables = []
 		self.link_tables = []
+		self.linked_table_aliases = {}
 		self.linked_table_counter = Counter()
 		self.conditions = []
 		self.or_conditions = []
@@ -79,7 +80,7 @@ class DatabaseQuery:
 
 	@property
 	def query_tables(self):
-		return self.tables + [d.table_name for d in self.link_tables]
+		return self.tables + [d.table_alias for d in self.link_tables]
 
 	def execute(
 		self,
@@ -508,6 +509,7 @@ class DatabaseQuery:
 			table_name=f"`tab{doctype}`",
 			table_alias=f"`tab{doctype}_{self.linked_table_counter[doctype]}`",
 		)
+		self.linked_table_aliases[linked_table.table_alias.replace("`", "")] = linked_table.table_name
 		self.link_tables.append(linked_table)
 		return linked_table
 
@@ -657,7 +659,12 @@ class DatabaseQuery:
 			# handle child / joined table fields
 			elif "." in field:
 				table, column = column.split(".", 1)
-				ch_doctype = table.replace("`", "").replace("tab", "", 1)
+				ch_doctype = table
+
+				if ch_doctype in self.linked_table_aliases:
+					ch_doctype = self.linked_table_aliases[ch_doctype]
+
+				ch_doctype = ch_doctype.replace("`", "").replace("tab", "", 1)
 
 				if wrap_grave_quotes(table) in self.query_tables:
 					permitted_child_table_fields = get_permitted_fields(

From 1c86d6431f8a397b1d9364eaa17a1c1ea2f764de Mon Sep 17 00:00:00 2001
From: Devin Slauenwhite <devin.slauenwhite@gmail.com>
Date: Fri, 3 Mar 2023 11:46:57 -0500
Subject: [PATCH 05/60] fix: don't join on parenttype

---
 frappe/model/db_query.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 4821f6558e..17935c226c 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -488,7 +488,7 @@ class DatabaseQuery:
 					table_name = table_name[13:]
 				if not table_name[0] == "`":
 					table_name = f"`{table_name}`"
-				if table_name not in self.query_tables:
+				if table_name not in self.query_tables and table_name not in self.linked_table_aliases.values():
 					self.append_table(table_name)
 
 	def append_table(self, table_name):

From f17e52088f988490bb7cf7c7586ed652ae20c3e3 Mon Sep 17 00:00:00 2001
From: Devin Slauenwhite <devin.slauenwhite@gmail.com>
Date: Fri, 3 Mar 2023 11:59:05 -0500
Subject: [PATCH 06/60] chore: linter

---
 frappe/model/db_query.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 17935c226c..394d3b8ccb 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -488,7 +488,9 @@ class DatabaseQuery:
 					table_name = table_name[13:]
 				if not table_name[0] == "`":
 					table_name = f"`{table_name}`"
-				if table_name not in self.query_tables and table_name not in self.linked_table_aliases.values():
+				if (
+					table_name not in self.query_tables and table_name not in self.linked_table_aliases.values()
+				):
 					self.append_table(table_name)
 
 	def append_table(self, table_name):

From 4ee5c015c35fcb450e7f751edd5e8b533b974d02 Mon Sep 17 00:00:00 2001
From: Dirk van der Laarse <dirk@laarse.co.za>
Date: Wed, 14 Jun 2023 12:45:49 +0000
Subject: [PATCH 07/60] feat: setting for force web capture mode for camera
 uploads

---
 cypress/integration/control_attach.js         | 45 +++++++++++++++++++
 .../system_settings/system_settings.json      |  9 +++-
 frappe/public/js/frappe/ui/capture.js         | 11 +++--
 3 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/cypress/integration/control_attach.js b/cypress/integration/control_attach.js
index 96b8c73b6e..d6086c9ed1 100644
--- a/cypress/integration/control_attach.js
+++ b/cypress/integration/control_attach.js
@@ -92,4 +92,49 @@ context("Attach Control", () => {
 		cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click();
 		cy.click_modal_primary_button("Yes");
 	});
+
+	it('Checking that "Camera" button in the "Attach" fieldtype does show if camera is available', () => {
+		//Navigating to the new form for the newly created doctype
+		let doctype = "Test Attach Control"
+		let dt_in_route = doctype.toLowerCase().replace(/ /g, "-");
+		cy.visit(`/app/${dt_in_route}/new`, {
+			onBeforeLoad (win) {
+			  // Mock "window.navigator.mediaDevices" property
+			  // to return mock mediaDevices object
+			  win.navigator.mediaDevices = {
+				ondevicechange: null
+			  }
+			}
+		  });
+		cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`);
+		cy.get("body").should("have.attr", "data-ajax-state", "complete");
+
+		//Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype
+		cy.findByRole("button", { name: "Attach" }).click();
+
+		//Clicking on "Camera" button
+		cy.findByRole("button", { name: "Camera" }).should("exist");
+
+	});
+
+	it('Checking that "Camera" button in the "Attach" fieldtype does not show if no camera is available', () => {
+		//Navigating to the new form for the newly created doctype
+		let doctype = "Test Attach Control"
+		let dt_in_route = doctype.toLowerCase().replace(/ /g, "-");
+		cy.visit(`/app/${dt_in_route}/new`, {
+			onBeforeLoad (win) {
+				// Delete "window.navigator.mediaDevices" property
+				delete win.navigator.mediaDevices
+			}
+			});
+		cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`);
+		cy.get("body").should("have.attr", "data-ajax-state", "complete");
+
+		//Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype
+		cy.findByRole("button", { name: "Attach" }).click();
+
+		//Clicking on "Camera" button
+		cy.findByRole("button", { name: "Camera" }).should("not.exist");
+			
+	});
 });
diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json
index 5efe87da25..5d88a3e466 100644
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@@ -563,12 +563,19 @@
    "fieldtype": "Link",
    "label": "Reset Password Template",
    "options": "Email Template"
+  },
+  {
+   "default": "0",
+   "description": "When uploading files, force the use of the web-based image capture. If this is unchecked, the default behavior is to use the mobile native camera when use from a mobile is detected.",
+   "fieldname": "force_web_capture_mode_for_uploads",
+   "fieldtype": "Check",
+   "label": "Force web capture mode for uploads"
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2023-05-25 13:02:54.808773",
+ "modified": "2023-06-14 11:19:33.429196",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",
diff --git a/frappe/public/js/frappe/ui/capture.js b/frappe/public/js/frappe/ui/capture.js
index bc7ce34149..72841885a0 100644
--- a/frappe/public/js/frappe/ui/capture.js
+++ b/frappe/public/js/frappe/ui/capture.js
@@ -76,11 +76,16 @@ frappe.ui.Capture = class {
 	show() {
 		this.build_dialog();
 
-		if (frappe.is_mobile()) {
-			this.show_for_mobile();
-		} else {
+		if (frappe.boot.sysdefaults.force_web_capture_mode_for_uploads) {
 			this.show_for_desktop();
 		}
+		else {
+			if (frappe.is_mobile()) {
+				this.show_for_mobile();
+			} else {
+				this.show_for_desktop();
+			}
+		}
 	}
 
 	build_dialog() {

From 93519703335a92b5e7be9acf271aa60ad4830ca9 Mon Sep 17 00:00:00 2001
From: Dirk van der Laarse <dirk@laarse.co.za>
Date: Wed, 14 Jun 2023 16:01:48 +0200
Subject: [PATCH 08/60] fix: cast sysdefaults setting to int

---
 frappe/public/js/frappe/ui/capture.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/ui/capture.js b/frappe/public/js/frappe/ui/capture.js
index 72841885a0..8309ffeeb8 100644
--- a/frappe/public/js/frappe/ui/capture.js
+++ b/frappe/public/js/frappe/ui/capture.js
@@ -76,7 +76,7 @@ frappe.ui.Capture = class {
 	show() {
 		this.build_dialog();
 
-		if (frappe.boot.sysdefaults.force_web_capture_mode_for_uploads) {
+		if (cint(frappe.boot.sysdefaults.force_web_capture_mode_for_uploads)) {
 			this.show_for_desktop();
 		}
 		else {

From ac8466d3d8b217a30b79bbb6eda2c99be2a5e834 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Wed, 19 Jul 2023 10:27:41 +0200
Subject: [PATCH 09/60] refactor(Geolocation): customize_draw_controls

Extract method `customize_draw_controls` from `bind_leaflet_map`
---
 frappe/public/js/frappe/form/controls/geolocation.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js
index 84b1989400..acff3d6ce8 100644
--- a/frappe/public/js/frappe/form/controls/geolocation.js
+++ b/frappe/public/js/frappe/form/controls/geolocation.js
@@ -35,6 +35,7 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f
 	}
 
 	make_map(value) {
+		this.customize_draw_controls();
 		this.bind_leaflet_map();
 		if (this.disabled) {
 			this.map.dragging.disable();
@@ -113,7 +114,7 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f
 	 */
 	on_each_feature(feature, layer) {}
 
-	bind_leaflet_map() {
+	customize_draw_controls() {
 		const circleToGeoJSON = L.Circle.prototype.toGeoJSON;
 		L.Circle.include({
 			toGeoJSON: function () {
@@ -138,6 +139,9 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f
 		});
 
 		L.Icon.Default.imagePath = "/assets/frappe/images/leaflet/";
+	}
+
+	bind_leaflet_map() {
 		this.map = L.map(this.map_id);
 		this.map.setView(frappe.utils.map_defaults.center, frappe.utils.map_defaults.zoom);
 

From 819f18acb47240a3bb25ca11139d051ca4b9f790 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Wed, 19 Jul 2023 10:32:23 +0200
Subject: [PATCH 10/60] refactor(Geolocation): image path via map defaults

Move the default path for icons to map_defaults instead of hardcoding it
in the control
---
 frappe/public/js/frappe/form/controls/geolocation.js | 2 +-
 frappe/public/js/frappe/utils/utils.js               | 1 +
 frappe/public/js/frappe/views/map/map_view.js        | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js
index acff3d6ce8..10ac21a5fb 100644
--- a/frappe/public/js/frappe/form/controls/geolocation.js
+++ b/frappe/public/js/frappe/form/controls/geolocation.js
@@ -138,7 +138,7 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f
 			},
 		});
 
-		L.Icon.Default.imagePath = "/assets/frappe/images/leaflet/";
+		L.Icon.Default.imagePath = frappe.utils.map_defaults.image_path;
 	}
 
 	bind_leaflet_map() {
diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js
index 4a8e519978..e4cae790f2 100644
--- a/frappe/public/js/frappe/utils/utils.js
+++ b/frappe/public/js/frappe/utils/utils.js
@@ -1203,6 +1203,7 @@ Object.assign(frappe.utils, {
 			attribution:
 				'&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
 		},
+		image_path: "/assets/frappe/images/leaflet/",
 	},
 
 	icon(icon_name, size = "sm", icon_class = "", icon_style = "", svg_class = "") {
diff --git a/frappe/public/js/frappe/views/map/map_view.js b/frappe/public/js/frappe/views/map/map_view.js
index e9bd71e3fc..b54938bc87 100644
--- a/frappe/public/js/frappe/views/map/map_view.js
+++ b/frappe/public/js/frappe/views/map/map_view.js
@@ -32,7 +32,7 @@ frappe.views.MapView = class MapView extends frappe.views.ListView {
 
 		this.$result.html(`<div id="${this.map_id}" class="map-view-container"></div>`);
 
-		L.Icon.Default.imagePath = "/assets/frappe/images/leaflet/";
+		L.Icon.Default.imagePath = frappe.utils.map_defaults.image_path;
 		this.map = L.map(this.map_id).setView(
 			frappe.utils.map_defaults.center,
 			frappe.utils.map_defaults.zoom

From 5ed4e45f525e2fa8a576df1be4630ba365250d48 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Wed, 19 Jul 2023 10:36:19 +0200
Subject: [PATCH 11/60] refactor(Geolocation): provide less utils

---
 frappe/public/js/frappe/form/controls/geolocation.js | 2 +-
 frappe/public/js/frappe/views/map/map_view.js        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js
index 10ac21a5fb..56a8d2a073 100644
--- a/frappe/public/js/frappe/form/controls/geolocation.js
+++ b/frappe/public/js/frappe/form/controls/geolocation.js
@@ -1,4 +1,4 @@
-frappe.provide("frappe.utils.utils");
+frappe.provide("frappe.utils");
 
 frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.form.ControlData {
 	static horizontal = false;
diff --git a/frappe/public/js/frappe/views/map/map_view.js b/frappe/public/js/frappe/views/map/map_view.js
index b54938bc87..b86faf3edc 100644
--- a/frappe/public/js/frappe/views/map/map_view.js
+++ b/frappe/public/js/frappe/views/map/map_view.js
@@ -1,7 +1,7 @@
 /**
  * frappe.views.MapView
  */
-frappe.provide("frappe.utils.utils");
+frappe.provide("frappe.utils");
 frappe.provide("frappe.views");
 
 frappe.views.MapView = class MapView extends frappe.views.ListView {

From 8ff913b8adc052481cfbecf004e59fc35e96c97d Mon Sep 17 00:00:00 2001
From: David Arnold <david.arnold@iohk.io>
Date: Sun, 23 Jul 2023 19:32:04 -0500
Subject: [PATCH 12/60] build(deps): update bleach 3.3.0 -> 6.0.0

---
 frappe/utils/html_utils.py | 92 ++++++++++++++++++++------------------
 pyproject.toml             |  2 +-
 2 files changed, 50 insertions(+), 44 deletions(-)

diff --git a/frappe/utils/html_utils.py b/frappe/utils/html_utils.py
index a109af798b..d3c101349f 100644
--- a/frappe/utils/html_utils.py
+++ b/frappe/utils/html_utils.py
@@ -19,13 +19,14 @@ EMOJI_PATTERN = re.compile(
 
 def clean_html(html):
 	import bleach
+	from bleach.css_sanitizer import CSSSanitizer
 
 	if not isinstance(html, str):
 		return html
 
 	return bleach.clean(
 		clean_script_and_style(html),
-		tags=[
+		tags={
 			"div",
 			"p",
 			"br",
@@ -42,9 +43,9 @@ def clean_html(html):
 			"tbody",
 			"td",
 			"tr",
-		],
+		},
 		attributes=[],
-		styles=["color", "border", "border-color"],
+		css_sanitizer=CSSSanitizer(allowed_css_properties=["color", "border", "border-color"]),
 		strip=True,
 		strip_comments=True,
 	)
@@ -52,44 +53,13 @@ def clean_html(html):
 
 def clean_email_html(html):
 	import bleach
+	from bleach.css_sanitizer import CSSSanitizer
 
 	if not isinstance(html, str):
 		return html
 
-	return bleach.clean(
-		clean_script_and_style(html),
-		tags=[
-			"div",
-			"p",
-			"br",
-			"ul",
-			"ol",
-			"li",
-			"strong",
-			"b",
-			"em",
-			"i",
-			"u",
-			"a",
-			"table",
-			"thead",
-			"tbody",
-			"td",
-			"tr",
-			"th",
-			"pre",
-			"code",
-			"h1",
-			"h2",
-			"h3",
-			"h4",
-			"h5",
-			"h6",
-			"button",
-			"img",
-		],
-		attributes=["border", "colspan", "rowspan", "src", "href", "style", "id"],
-		styles=[
+	css_sanitizer = CSSSanitizer(
+		allowed_css_properties=[
 			"color",
 			"border-color",
 			"width",
@@ -121,7 +91,43 @@ def clean_email_html(html):
 			"text-align",
 			"vertical-align",
 			"display",
-		],
+		]
+	)
+
+	return bleach.clean(
+		clean_script_and_style(html),
+		tags={
+			"div",
+			"p",
+			"br",
+			"ul",
+			"ol",
+			"li",
+			"strong",
+			"b",
+			"em",
+			"i",
+			"u",
+			"a",
+			"table",
+			"thead",
+			"tbody",
+			"td",
+			"tr",
+			"th",
+			"pre",
+			"code",
+			"h1",
+			"h2",
+			"h3",
+			"h4",
+			"h5",
+			"h6",
+			"button",
+			"img",
+		},
+		attributes=["border", "colspan", "rowspan", "src", "href", "style", "id"],
+		css_sanitizer=css_sanitizer,
 		protocols=["cid", "http", "https", "mailto", "data"],
 		strip=True,
 		strip_comments=True,
@@ -146,6 +152,7 @@ def sanitize_html(html, linkify=False):
 	Does not sanitize JSON, as it could lead to future problems
 	"""
 	import bleach
+	from bleach.css_sanitizer import CSSSanitizer
 	from bs4 import BeautifulSoup
 
 	if not isinstance(html, str):
@@ -170,17 +177,16 @@ def sanitize_html(html, linkify=False):
 		return name in acceptable_attributes
 
 	attributes = {"*": attributes_filter, "svg": svg_attributes}
-	styles = bleach_allowlist.all_styles
-	strip_comments = False
+	css_sanitizer = CSSSanitizer(allowed_css_properties=bleach_allowlist.all_styles)
 
 	# returns html with escaped tags, escaped orphan >, <, etc.
 	escaped_html = bleach.clean(
 		html,
 		tags=tags,
 		attributes=attributes,
-		styles=styles,
-		strip_comments=strip_comments,
-		protocols=["cid", "http", "https", "mailto"],
+		css_sanitizer=css_sanitizer,
+		strip_comments=False,
+		protocols={"cid", "http", "https", "mailto"},
 	)
 
 	return escaped_html
diff --git a/pyproject.toml b/pyproject.toml
index b2b890c24c..33e0ff9f1a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,7 +28,7 @@ dependencies = [
     "Whoosh~=2.7.4",
     "beautifulsoup4~=4.12.2",
     "bleach-allowlist~=1.0.3",
-    "bleach~=3.3.0",
+    "bleach[css]~=6.0.0",
     "cairocffi==1.5.1",
     "chardet~=5.1.0",
     "croniter~=1.3.15",

From a4385be70ce0f514a4d4b8712d29e04347597ab6 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Mon, 24 Jul 2023 16:02:22 +0200
Subject: [PATCH 13/60] fix: pass email as value

---
 frappe/contacts/doctype/contact/test_contact.py | 2 ++
 frappe/email/__init__.py                        | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/frappe/contacts/doctype/contact/test_contact.py b/frappe/contacts/doctype/contact/test_contact.py
index 18f0d78732..b5f1c4bdf8 100644
--- a/frappe/contacts/doctype/contact/test_contact.py
+++ b/frappe/contacts/doctype/contact/test_contact.py
@@ -49,11 +49,13 @@ class TestContact(FrappeTestCase):
 		# First time from database
 		results = get_contact_list("_Test Supplier")
 		self.assertEqual(results[0].label, "test_contact@example.com")
+		self.assertEqual(results[0].value, "test_contact@example.com")
 		self.assertEqual(results[0].description, "_Test Contact For _Test Supplier")
 
 		# Second time from cache
 		results = get_contact_list("_Test Supplier")
 		self.assertEqual(results[0].label, "test_contact@example.com")
+		self.assertEqual(results[0].value, "test_contact@example.com")
 		self.assertEqual(results[0].description, "_Test Contact For _Test Supplier")
 
 
diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py
index 463f54d7e0..666c726942 100644
--- a/frappe/email/__init__.py
+++ b/frappe/email/__init__.py
@@ -16,7 +16,7 @@ def get_contact_list(txt, page_length=20) -> list[dict]:
 	if cached_contacts := get_cached_contacts(txt):
 		return cached_contacts[:page_length]
 
-	fields = ["name", "first_name", "middle_name", "last_name", "company_name"]
+	fields = ["first_name", "middle_name", "last_name", "company_name"]
 	contacts = frappe.get_list(
 		"Contact",
 		fields=fields + ["`tabContact Email`.email_id"],
@@ -33,7 +33,7 @@ def get_contact_list(txt, page_length=20) -> list[dict]:
 	# https://github.com/frappe/frappe/blob/6c6a89bcdd9454060a1333e23b855d0505c9ebc2/frappe/public/js/frappe/form/controls/autocomplete.js#L29-L35
 	result = [
 		frappe._dict(
-			value=d.name,
+			value=d.email_id,
 			label=d.email_id,
 			description=get_full_name(d.first_name, d.middle_name, d.last_name, d.company_name),
 		)

From f775c014e49b5e824827ee1e81cd4e4c7c20ebe8 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Tue, 25 Jul 2023 13:06:49 +0530
Subject: [PATCH 14/60] chore: simple codeowners

[skip ci]
---
 CODEOWNERS | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index 861016710a..e636e6c9fb 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -4,9 +4,3 @@
 # the repo. Unless a later match takes precedence,
 
 *                 @frappe/frappe-review-team
-templates/        @surajshetty3416
-www/              @surajshetty3416
-patches/          @surajshetty3416
-data_import*      @netchampfaris
-core/             @surajshetty3416
-workspace         @shariquerik

From 7780670ae44c5359e6e404075d52e65600e960bd Mon Sep 17 00:00:00 2001
From: David Arnold <david.arnold@iohk.io>
Date: Tue, 25 Jul 2023 03:37:46 -0500
Subject: [PATCH 15/60] build(deps): update node redis client to v4 (#21797)

* build(deps): update redis client to v4 in legacy mode

* fix: node17+ - prefer ipv4

* chore: use redis client v4 api (async) and adapt error handling

* fix: timeout by exiting if not in watch mode

* fix: parse message before republishing

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 esbuild/esbuild.js | 67 ++++++++++++++++++++++++----------------------
 node_utils.js      |  8 +++++-
 package.json       |  2 +-
 realtime/index.js  | 29 ++++++++++----------
 yarn.lock          | 61 +++++++++++++++++------------------------
 5 files changed, 82 insertions(+), 85 deletions(-)

diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js
index 8937a03216..e4a226281b 100644
--- a/esbuild/esbuild.js
+++ b/esbuild/esbuild.js
@@ -89,12 +89,10 @@ const NODE_PATHS = [].concat(
 	app_list.map((app) => path.resolve(get_app_path(app), "..")).filter(fs.existsSync)
 );
 
-execute()
-	.then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS))
-	.catch((e) => {
-		console.error(e);
-		process.exit(1);
-	});
+execute().catch((e) => {
+	console.error(e);
+	process.exit(1);
+});
 
 if (WATCH_MODE) {
 	// listen for open files in editor event
@@ -127,6 +125,10 @@ async function execute() {
 	for (const result of results) {
 		await write_assets_json(result.metafile);
 	}
+	RUN_BUILD_COMMAND && run_build_command_for_apps(APPS);
+	if (!WATCH_MODE) {
+		process.exit(0);
+	}
 }
 
 function build_assets_for_apps(apps, files) {
@@ -418,18 +420,18 @@ async function write_assets_json(metafile) {
 	};
 }
 
-function update_assets_json_in_cache() {
+async function update_assets_json_in_cache() {
 	// update assets_json cache in redis, so that it can be read directly by python
-	return new Promise((resolve) => {
-		let client = get_redis_subscriber("redis_cache");
-		// handle error event to avoid printing stack traces
-		client.on("error", (_) => {
-			log_warn("Cannot connect to redis_cache to update assets_json");
-		});
-		client.del("assets_json", (err) => {
-			client.unref();
-			resolve();
-		});
+	let client = get_redis_subscriber("redis_cache");
+	// handle error event to avoid printing stack traces
+	try {
+		await client.connect();
+	} catch (e) {
+		log_warn("Cannot connect to redis_cache to update assets_json");
+	}
+	client.del("assets_json", (err) => {
+		client.unref();
+		resolve();
 	});
 }
 
@@ -458,9 +460,11 @@ function run_build_command_for_apps(apps) {
 async function notify_redis({ error, success, changed_files }) {
 	// notify redis which in turns tells socketio to publish this to browser
 	let subscriber = get_redis_subscriber("redis_queue");
-	subscriber.on("error", (_) => {
+	try {
+		await subscriber.connect();
+	} catch (e) {
 		log_warn("Cannot connect to redis_queue for browser events");
-	});
+	}
 
 	let payload = null;
 	if (error) {
@@ -483,7 +487,7 @@ async function notify_redis({ error, success, changed_files }) {
 		};
 	}
 
-	subscriber.publish(
+	await subscriber.publish(
 		"events",
 		JSON.stringify({
 			event: "build_event",
@@ -492,21 +496,20 @@ async function notify_redis({ error, success, changed_files }) {
 	);
 }
 
-function open_in_editor() {
+async function open_in_editor() {
 	let subscriber = get_redis_subscriber("redis_queue");
-	subscriber.on("error", (_) => {
+	try {
+		await subscriber.connect();
+	} catch (e) {
 		log_warn("Cannot connect to redis_queue for open_in_editor events");
+	}
+	subscriber.subscribe("open_in_editor", (file) => {
+		file = JSON.parse(file);
+		let file_path = path.resolve(file.file);
+		log("Opening file in editor:", file_path);
+		let launch = require("launch-editor");
+		launch(`${file_path}:${file.line}:${file.column}`);
 	});
-	subscriber.on("message", (event, file) => {
-		if (event === "open_in_editor") {
-			file = JSON.parse(file);
-			let file_path = path.resolve(file.file);
-			log("Opening file in editor:", file_path);
-			let launch = require("launch-editor");
-			launch(`${file_path}:${file.line}:${file.column}`);
-		}
-	});
-	subscriber.subscribe("open_in_editor");
 }
 
 function get_rebuilt_assets(prev_assets, new_assets) {
diff --git a/node_utils.js b/node_utils.js
index 14b6f3576b..38c9b1c1cc 100644
--- a/node_utils.js
+++ b/node_utils.js
@@ -1,8 +1,14 @@
 const fs = require("fs");
 const path = require("path");
-const redis = require("redis");
+const redis = require("@redis/client");
 const bench_path = path.resolve(__dirname, "..", "..");
 
+const dns = require("dns");
+
+// Since node17, node resolves to ipv6 unless system is configured otherwise.
+// In Frappe context using ipv4 - 127.0.0.1 is fine.
+dns.setDefaultResultOrder("ipv4first");
+
 function get_conf() {
 	// defaults
 	var conf = {
diff --git a/package.json b/package.json
index 2cd1e6894e..0794ae5d1d 100644
--- a/package.json
+++ b/package.json
@@ -22,6 +22,7 @@
   "dependencies": {
     "@editorjs/editorjs": "^2.26.3",
     "@frappe/esbuild-plugin-postcss2": "^0.1.3",
+    "@redis/client": "^1.5.8",
     "@vue-flow/background": "^1.1.0",
     "@vue-flow/core": "^1.16.2",
     "@vue/component-compiler": "^4.2.4",
@@ -63,7 +64,6 @@
     "quill-image-resize": "^3.0.9",
     "quill-magic-url": "^3.0.0",
     "qz-tray": "^2.0.8",
-    "redis": "^3.1.1",
     "rtlcss": "^4.0.0",
     "sass": "^1.63.0",
     "showdown": "^2.1.0",
diff --git a/realtime/index.js b/realtime/index.js
index ecc6063ca4..c60ec66330 100644
--- a/realtime/index.js
+++ b/realtime/index.js
@@ -27,7 +27,8 @@ function on_connection(socket) {
 	frappe_handlers(realtime, socket);
 
 	// ESBUild "open in editor" on error
-	socket.on("open_in_editor", (data) => {
+	socket.on("open_in_editor", async (data) => {
+		await subscriber.connect();
 		subscriber.publish("open_in_editor", JSON.stringify(data));
 	});
 }
@@ -38,19 +39,19 @@ realtime.on("connection", on_connection);
 // Consume events sent from python via redis pub-sub channel.
 const subscriber = get_redis_subscriber();
 
-subscriber.on("message", function (_channel, message) {
-	message = JSON.parse(message);
-
-	let namespace = "/" + message.namespace;
-	if (message.room) {
-		io.of(namespace).to(message.room).emit(message.event, message.message);
-	} else {
-		// publish to ALL sites only used for things like build event.
-		realtime.emit(message.event, message.message);
-	}
-});
-
-subscriber.subscribe("events");
+(async () => {
+	await subscriber.connect();
+	subscriber.subscribe("events", (message) => {
+		message = JSON.parse(message);
+		let namespace = "/" + message.namespace;
+		if (message.room) {
+			io.of(namespace).to(message.room).emit(message.event, message.message);
+		} else {
+			// publish to ALL sites only used for things like build event.
+			realtime.emit(message.event, message.message);
+		}
+	});
+})();
 // =======================
 
 let port = conf.socketio_port;
diff --git a/yarn.lock b/yarn.lock
index e816b29656..f632157bbe 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -81,6 +81,15 @@
     "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
+"@redis/client@^1.5.8":
+  version "1.5.8"
+  resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.8.tgz#a375ba7861825bd0d2dc512282b8bff7b98dbcb1"
+  integrity sha512-xzElwHIO6rBAqzPeVnCzgvrnBEcFL1P0w8P65VNLRkdVW8rOE58f52hdj0BDgmsdOm4f1EoXPZtH4Fh7M/qUpw==
+  dependencies:
+    cluster-key-slot "1.1.2"
+    generic-pool "3.9.0"
+    yallist "4.0.0"
+
 "@socket.io/component-emitter@~3.1.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553"
@@ -592,6 +601,11 @@ clone@^2.1.1, clone@^2.1.2:
   resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
   integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==
 
+cluster-key-slot@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
+  integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
+
 color-convert@^1.9.0:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
@@ -973,11 +987,6 @@ delayed-stream@~1.0.0:
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
   integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==
 
-denque@^1.5.0:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf"
-  integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
-
 dezalgo@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81"
@@ -1457,6 +1466,11 @@ generic-names@^4.0.0:
   dependencies:
     loader-utils "^3.2.0"
 
+generic-pool@3.9.0:
+  version "3.9.0"
+  resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
+  integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==
+
 get-caller-file@^2.0.5:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
@@ -2874,33 +2888,6 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
-redis-commands@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
-  integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==
-
-redis-errors@^1.0.0, redis-errors@^1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
-  integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==
-
-redis-parser@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
-  integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==
-  dependencies:
-    redis-errors "^1.0.0"
-
-redis@^3.1.1:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/redis/-/redis-3.1.2.tgz#766851117e80653d23e0ed536254677ab647638c"
-  integrity sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==
-  dependencies:
-    denque "^1.5.0"
-    redis-commands "^1.7.0"
-    redis-errors "^1.2.0"
-    redis-parser "^3.0.0"
-
 regexp.prototype.flags@^1.2.0, regexp.prototype.flags@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz#fe7ce25e7e4cca8db37b6634c8a2c7009199b9cb"
@@ -3479,16 +3466,16 @@ y18n@^5.0.5:
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
   integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
 
+yallist@4.0.0, yallist@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
+  integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
+
 yallist@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
   integrity sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==
 
-yallist@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
-  integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
-
 yaml@^1.10.2:
   version "1.10.2"
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"

From c0d472e95ccf82986d4369247fcaa35fcbff6564 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 25 Jul 2023 16:21:59 +0530
Subject: [PATCH 16/60] refactor: Remove unnecessary code

---
 frappe/utils/html_utils.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/frappe/utils/html_utils.py b/frappe/utils/html_utils.py
index d3c101349f..ca1cbc3a74 100644
--- a/frappe/utils/html_utils.py
+++ b/frappe/utils/html_utils.py
@@ -19,7 +19,6 @@ EMOJI_PATTERN = re.compile(
 
 def clean_html(html):
 	import bleach
-	from bleach.css_sanitizer import CSSSanitizer
 
 	if not isinstance(html, str):
 		return html
@@ -45,7 +44,6 @@ def clean_html(html):
 			"tr",
 		},
 		attributes=[],
-		css_sanitizer=CSSSanitizer(allowed_css_properties=["color", "border", "border-color"]),
 		strip=True,
 		strip_comments=True,
 	)

From dff950a56d505899043a4f8b8499eb662087d092 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 25 Jul 2023 18:20:00 +0530
Subject: [PATCH 17/60] fix: Remove unused `resolve` in code

fixes: https://github.com/frappe/frappe/actions/runs/5655856445/job/15321775151?pr=21782
---
 esbuild/esbuild.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js
index e4a226281b..e22613e0e9 100644
--- a/esbuild/esbuild.js
+++ b/esbuild/esbuild.js
@@ -431,7 +431,6 @@ async function update_assets_json_in_cache() {
 	}
 	client.del("assets_json", (err) => {
 		client.unref();
-		resolve();
 	});
 }
 

From 766e0b056b267defaffea1f3dd2ce7acb530c7be Mon Sep 17 00:00:00 2001
From: Corentin Flr <10946971+cogk@users.noreply.github.com>
Date: Mon, 24 Jul 2023 16:12:48 +0200
Subject: [PATCH 18/60] fix(Form Builder): Fix label text font size

---
 .../public/js/form_builder/components/controls/CheckControl.vue | 2 +-
 .../public/js/form_builder/components/controls/DataControl.vue  | 2 +-
 .../js/form_builder/components/controls/SelectControl.vue       | 2 +-
 .../public/js/form_builder/components/controls/TextControl.vue  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/frappe/public/js/form_builder/components/controls/CheckControl.vue b/frappe/public/js/form_builder/components/controls/CheckControl.vue
index 023bcebf66..fbdb76396d 100644
--- a/frappe/public/js/form_builder/components/controls/CheckControl.vue
+++ b/frappe/public/js/form_builder/components/controls/CheckControl.vue
@@ -6,7 +6,7 @@ let slots = useSlots();
 </script>
 
 <template>
-	<div class="control checkbox" :class="{ editable: slots.label }">
+	<div class="control frappe-control checkbox" :class="{ editable: slots.label }">
 		<!-- checkbox -->
 		<label v-if="slots.label" class="field-controls">
 			<div class="checkbox">
diff --git a/frappe/public/js/form_builder/components/controls/DataControl.vue b/frappe/public/js/form_builder/components/controls/DataControl.vue
index 75e86720d1..53433100c4 100644
--- a/frappe/public/js/form_builder/components/controls/DataControl.vue
+++ b/frappe/public/js/form_builder/components/controls/DataControl.vue
@@ -32,7 +32,7 @@ if (props.df.fieldtype === "Icon") {
 			<slot name="label" />
 			<slot name="actions" />
 		</div>
-		<div v-else class="label" :class="{ reqd: df.reqd }">{{ df.label }}</div>
+		<div v-else class="control-label label" :class="{ reqd: df.reqd }">{{ df.label }}</div>
 
 		<!-- data input -->
 		<input
diff --git a/frappe/public/js/form_builder/components/controls/SelectControl.vue b/frappe/public/js/form_builder/components/controls/SelectControl.vue
index 858bd8a192..c7279efb35 100644
--- a/frappe/public/js/form_builder/components/controls/SelectControl.vue
+++ b/frappe/public/js/form_builder/components/controls/SelectControl.vue
@@ -83,7 +83,7 @@ watch(() => props.df.options, () => {
 </script>
 
 <template>
-	<div v-if="slots.label" class="control" :class="{ editable: slots.label }">
+	<div v-if="slots.label" class="control frappe-control" :class="{ editable: slots.label }">
 		<!-- label -->
 		<div class="field-controls">
 			<slot name="label" />
diff --git a/frappe/public/js/form_builder/components/controls/TextControl.vue b/frappe/public/js/form_builder/components/controls/TextControl.vue
index 38e9347d4f..5e0aa382d7 100644
--- a/frappe/public/js/form_builder/components/controls/TextControl.vue
+++ b/frappe/public/js/form_builder/components/controls/TextControl.vue
@@ -21,7 +21,7 @@ let height = computed(() => {
 			<slot name="label" />
 			<slot name="actions" />
 		</div>
-		<div v-else class="label">{{ df.label }}</div>
+		<div v-else class="control-label label">{{ df.label }}</div>
 
 		<!-- textarea input -->
 		<textarea

From c87a6e8ff07e7368a6362d3cc4805c713db4261c Mon Sep 17 00:00:00 2001
From: Corentin Flr <10946971+cogk@users.noreply.github.com>
Date: Mon, 24 Jul 2023 16:15:00 +0200
Subject: [PATCH 19/60] fix(Form Builder): Handle sort_options in Select

---
 .../js/form_builder/components/controls/SelectControl.vue   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/form_builder/components/controls/SelectControl.vue b/frappe/public/js/form_builder/components/controls/SelectControl.vue
index c7279efb35..9ad9922231 100644
--- a/frappe/public/js/form_builder/components/controls/SelectControl.vue
+++ b/frappe/public/js/form_builder/components/controls/SelectControl.vue
@@ -26,12 +26,16 @@ function get_options() {
 
 	if (props.df.fieldname == "fieldtype") {
 		if (!in_list(frappe.model.layout_fields, props.modelValue)) {
-			options = options && options.filter(opt => !in_list(frappe.model.layout_fields, opt.label));
+			options = options && options.filter(opt => !in_list(frappe.model.layout_fields, opt.value));
 		} else {
 			options = [{ label: __(props.modelValue), value: props.modelValue }];
 		}
 	}
 
+	if (props.df.sort_options) {
+		options.sort((a, b) => a.label.localeCompare(b.label));
+	}
+
 	return options;
 }
 

From c45fbce2faf5bdcb86ad0f96f59dfbbc61082134 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Wed, 26 Jul 2023 09:23:06 +0530
Subject: [PATCH 20/60] fix: ignore no args to email read url

Some client can strip this and it will cause error.
---
 frappe/email/doctype/newsletter/newsletter.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py
index 554ea79b08..09200e3635 100644
--- a/frappe/email/doctype/newsletter/newsletter.py
+++ b/frappe/email/doctype/newsletter/newsletter.py
@@ -411,7 +411,9 @@ def send_scheduled_email():
 
 
 @frappe.whitelist(allow_guest=True)
-def newsletter_email_read(recipient_email, reference_doctype, reference_name):
+def newsletter_email_read(recipient_email=None, reference_doctype=None, reference_name=None):
+	if not (recipient_email and reference_name):
+		return
 	verify_request()
 	try:
 		doc = frappe.get_cached_doc("Newsletter", reference_name)

From 5f46f98699eec422abb9acb77081ddd4b7731ff1 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Wed, 26 Jul 2023 14:31:03 +0530
Subject: [PATCH 21/60] ci: remove mysql (#21811)

---
 .github/helper/install_dependencies.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh
index b8da022665..574144b823 100644
--- a/.github/helper/install_dependencies.sh
+++ b/.github/helper/install_dependencies.sh
@@ -4,6 +4,7 @@ set -e
 echo "Setting Up System Dependencies..."
 
 sudo apt update
+sudo apt remove mysql-server mysql-client
 sudo apt install libcups2-dev redis-server mariadb-client-10.6
 
 install_wkhtmltopdf() {

From 89b9b64a552d55a771a821eb9eb74a60d4d261d2 Mon Sep 17 00:00:00 2001
From: Smit Vora <smitvora203@gmail.com>
Date: Wed, 26 Jul 2023 15:00:04 +0530
Subject: [PATCH 22/60] fix: installation error, cannot import name
 'DEFAULT_CIPHERS' from 'urllib3.util.ssl_' (#21810)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 16191610d2..b0cbe982af 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -74,7 +74,7 @@ dependencies = [
     "markdownify~=0.11.6",
 
     # integration dependencies
-    "boto3~=1.18.49",
+    "boto3~=1.28.10",
     "dropbox~=11.36.0",
     "google-api-python-client~=2.2.0",
     "google-auth-oauthlib~=0.4.4",

From 6cb51a536e85efdbb0794f067a04981fff96e3aa Mon Sep 17 00:00:00 2001
From: gavin <gavin18d@gmail.com>
Date: Thu, 27 Jul 2023 10:40:50 +0530
Subject: [PATCH 23/60] fix: `re` module cache settings - purge & cache size
 (#21808)

* fix: Remove re cache internals manipulation

* fix: Purge re cache after module loads

Empty cache would work better as we already got our pre-compiled
patterns at the top level of every module. This leaves the cache open
for dynamically generated patterns which are in better need of it. Over
time, workers would converge to this anyway. This change only reduces
the cache hit and eviction effort.

I'd improve this by executing `re.purge` on every module import but
complexity tradeoff lol. I'd prefer if re didn't cache patterns
generated by `re.compile` but I dont see this behaviour or any escape
hatches so this will have to do for now.
---
 frappe/__init__.py | 9 ++++-----
 frappe/app.py      | 4 ++++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index b3a5c54fa5..fd77930816 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -53,12 +53,8 @@ local = Local()
 cache = None
 STANDARD_USERS = ("Guest", "Administrator")
 
-_dev_server = int(sbool(os.environ.get("DEV_SERVER", False)))
 _qb_patched = {}
-re._MAXCACHE = (
-	50  # reduced from default 512 given we are already maintaining this on parent worker
-)
-
+_dev_server = int(sbool(os.environ.get("DEV_SERVER", False)))
 _tune_gc = bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True)))
 
 if _dev_server:
@@ -2450,3 +2446,6 @@ if _tune_gc:
 	# everything else.
 	g0, g1, g2 = gc.get_threshold()  # defaults are 700, 10, 10.
 	gc.set_threshold(g0 * 10, g1 * 2, g2 * 2)
+
+# Remove references to pattern that are pre-compiled and loaded to global scopes.
+re.purge()
diff --git a/frappe/app.py b/frappe/app.py
index cdb5d6ad21..137165c1e9 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -4,6 +4,7 @@
 import gc
 import logging
 import os
+import re
 
 from werkzeug.exceptions import HTTPException, NotFound
 from werkzeug.local import LocalManager
@@ -428,6 +429,9 @@ def serve(
 	)
 
 
+# Remove references to pattern that are pre-compiled and loaded to global scopes.
+re.purge()
+
 # Both Gunicorn and RQ use forking to spawn workers. In an ideal world, the fork should be sharing
 # most of the memory if there are no writes made to data because of Copy on Write, however,
 # python's GC is not CoW friendly and writes to data even if user-code doesn't. Specifically, the

From b4629d8c02df1aef4091427f6958bdc3ea6c1fbe Mon Sep 17 00:00:00 2001
From: Sagar Vora <sagar@resilient.tech>
Date: Thu, 27 Jul 2023 05:11:20 +0000
Subject: [PATCH 24/60] chore: strictly pin responses to `0.23.1` to avoid pip
 conflicts (#21816)

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index b0cbe982af..2afef9902f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -105,4 +105,4 @@ pyngrok = "~=6.0.0"
 unittest-xml-reporting = "~=3.2.0"
 watchdog = "~=3.0.0"
 hypothesis = "~=6.77.0"
-responses = "~=0.23.1"
+responses = "==0.23.1"

From 8930d4b5e1bb566da2952f6888cea1269f86d562 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 27 Jul 2023 10:58:20 +0530
Subject: [PATCH 25/60] perf: drop `ifnull` from `IS SET` filter (#21822)

- Kinda confuses query planner (idk why it's not smart enough to
  understand but there are probably edge cases where it can't be done)
- `null != null` and `'' != null` both yield `null` which is falsy and
  won't be shown in results.

Alternate fix to https://github.com/frappe/frappe/pull/21817
---
 frappe/model/db_query.py      | 11 ++++++++---
 frappe/tests/test_db_query.py |  5 +++++
 frappe/tests/test_perf.py     |  5 +++++
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 33c55a1691..534367b2fd 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -815,14 +815,19 @@ class DatabaseQuery:
 			elif f.operator.lower() == "is":
 				if f.value == "set":
 					f.operator = "!="
+					# Value can technically be null, but comparing with null will always be falsy
+					# Not using coalesce here is faster because indexes can be used.
+					# null != '' -> null ~ falsy
+					# '' != '' -> false
+					can_be_null = False
 				elif f.value == "not set":
 					f.operator = "="
+					fallback = "''"
+					can_be_null = True
 
 				value = ""
-				fallback = "''"
-				can_be_null = True
 
-				if "ifnull" not in column_name.lower():
+				if can_be_null and "ifnull" not in column_name.lower():
 					column_name = f"ifnull({column_name}, {fallback})"
 
 			elif df and df.fieldtype == "Date":
diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py
index ec835ffab4..90bb852148 100644
--- a/frappe/tests/test_db_query.py
+++ b/frappe/tests/test_db_query.py
@@ -704,6 +704,11 @@ class TestDBQuery(FrappeTestCase):
 		self.assertTrue({"name": "Prepared Report"} in res)
 		self.assertFalse({"name": "Property Setter"} in res)
 
+		frappe.db.set_value("DocType", "Property Setter", "autoname", None, update_modified=False)
+
+		res = DatabaseQuery("DocType").execute(filters={"autoname": ["is", "set"]})
+		self.assertFalse({"name": "Property Setter"} in res)
+
 	def test_set_field_tables(self):
 		# Tests _in_standard_sql_methods method in test_set_field_tables
 		# The following query will break if the above method is broken
diff --git a/frappe/tests/test_perf.py b/frappe/tests/test_perf.py
index 4350f1d31a..ed9470d4d2 100644
--- a/frappe/tests/test_perf.py
+++ b/frappe/tests/test_perf.py
@@ -158,3 +158,8 @@ class TestPerformance(FrappeTestCase):
 		frappe.get_list("User")
 		with self.assertQueryCount(1):
 			frappe.get_list("User")
+
+	def test_no_ifnull_checks(self):
+		query = frappe.get_all("DocType", {"autoname": ("is", "set")}, run=0).lower()
+		self.assertNotIn("coalesce", query)
+		self.assertNotIn("ifnull", query)

From 9902f36812446daed6f41a120f4e5e335214eabf Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Thu, 27 Jul 2023 13:22:38 +0530
Subject: [PATCH 26/60] fix: show full name instead of first/last name

---
 frappe/integrations/oauth2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py
index bf71a68aaa..724d3b32a7 100644
--- a/frappe/integrations/oauth2.py
+++ b/frappe/integrations/oauth2.py
@@ -103,7 +103,7 @@ def authorize(**kwargs):
 			else:
 				if "openid" in scopes:
 					scopes.remove("openid")
-					scopes.extend(["First Name", "Last Name", "Email", "Password", "User Image", "Roles"])
+					scopes.extend(["Full Name", "Email", "User Image", "Roles"])
 
 				# Show Allow/Deny screen.
 				response_html_params = frappe._dict(

From 99a4116725ee39aeb3c736e43181c5bacfa9c434 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Thu, 27 Jul 2023 13:23:24 +0530
Subject: [PATCH 27/60] fix: only render message with p tag if message exist

---
 frappe/www/message.html | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/www/message.html b/frappe/www/message.html
index c579228b78..9039e13186 100644
--- a/frappe/www/message.html
+++ b/frappe/www/message.html
@@ -26,7 +26,9 @@ html, body {
 	</h5>
 	<div class="page-card-body">
 	{% block message_body %}
-		<p>{{ message or "" }}</p>
+		{% if message %}
+		<p>{{ message }}</p>
+		{% endif %}
 		{% if primary_action %}
 		<div><a href='{{ primary_action or "/" }}' class='btn btn-primary btn-sm btn-block'>
 			{{ primary_label or _("Home") }}</a></div>

From d7c3e488197d3d96483c44029c0930e2de5afb47 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Thu, 27 Jul 2023 13:23:49 +0530
Subject: [PATCH 28/60] style: hide p if it is empty

---
 frappe/templates/styles/card_style.css | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/frappe/templates/styles/card_style.css b/frappe/templates/styles/card_style.css
index a9639d8133..96898847d2 100644
--- a/frappe/templates/styles/card_style.css
+++ b/frappe/templates/styles/card_style.css
@@ -75,6 +75,10 @@ button#update {
     margin-top: 30px;
 }
 
+.page-card p:empty {
+    display: none;
+}
+
 .page-card p {
     font-size: 14px;
 }

From c40faddac7b3f1b08ac558c4119392469f187484 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 27 Jul 2023 16:07:22 +0530
Subject: [PATCH 29/60] perf: skip reset_seen for new doc (#21832)

The query is fired but document doesn't exist yet, so it does nothing really.

[skip ci]
---
 frappe/model/document.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/frappe/model/document.py b/frappe/model/document.py
index 591576c962..3343a5dab8 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -1134,7 +1134,11 @@ class Document(BaseDocument):
 
 	def reset_seen(self):
 		"""Clear _seen property and set current user as seen"""
-		if getattr(self.meta, "track_seen", False) and not getattr(self.meta, "issingle", False):
+		if (
+			getattr(self.meta, "track_seen", False)
+			and not getattr(self.meta, "issingle", False)
+			and not self.is_new()
+		):
 			frappe.db.set_value(
 				self.doctype, self.name, "_seen", json.dumps([frappe.session.user]), update_modified=False
 			)

From 35c929afdbecf7bd04f5bea4b6c6812dbd8ffa3d Mon Sep 17 00:00:00 2001
From: Abdo <ruzaqi@gmail.com>
Date: Thu, 27 Jul 2023 14:51:57 +0300
Subject: [PATCH 30/60] refactor(money in words): translatable currency Name
 (#21672)

---
 frappe/utils/data.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/utils/data.py b/frappe/utils/data.py
index c42a741017..ca8f25c80a 100644
--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@@ -1353,12 +1353,12 @@ def money_in_words(
 
 	# 0.00
 	if main == "0" and fraction in ["00", "000"]:
-		out = "{} {}".format(main_currency, _("Zero"))
+		out = _(main_currency, context="Currency") + " " + _("Zero")
 	# 0.XX
 	elif main == "0":
 		out = _(in_words(fraction, in_million).title()) + " " + fraction_currency
 	else:
-		out = main_currency + " " + _(in_words(main, in_million).title())
+		out = _(main_currency, context="Currency") + " " + _(in_words(main, in_million).title())
 		if cint(fraction):
 			out = (
 				out

From 6e94cd2eb9a500a509331fcbe5eb55ca5c5148a4 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 27 Jul 2023 17:30:04 +0530
Subject: [PATCH 31/60] fix: Guess most likely exception source (#21827)

---
 frappe/__init__.py                            |  6 +--
 .../core/doctype/error_log/test_error_log.py  | 45 ++++++++++++++++++-
 frappe/public/js/frappe/request.js            |  9 +++-
 frappe/utils/error.py                         | 33 ++++++++++++++
 frappe/utils/response.py                      |  4 ++
 5 files changed, 91 insertions(+), 6 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index fd77930816..9a83443b02 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1460,13 +1460,11 @@ def get_all_apps(with_internal_apps=True, sites_path=None):
 
 
 @request_cache
-def get_installed_apps(*, _ensure_on_bench=False):
+def get_installed_apps(*, _ensure_on_bench=False) -> list[str]:
 	"""
 	Get list of installed apps in current site.
 
-	:param sort: [DEPRECATED] Sort installed apps based on the sequence in sites/apps.txt
-	:param frappe_last: [DEPRECATED] Keep frappe last. Do not use this, reverse the app list instead.
-	:param ensure_on_bench: Only return apps that are present on bench.
+	:param _ensure_on_bench: Only return apps that are present on bench.
 	"""
 	if getattr(flags, "in_install_db", True):
 		return []
diff --git a/frappe/core/doctype/error_log/test_error_log.py b/frappe/core/doctype/error_log/test_error_log.py
index 3f19a6dd0c..22eeea329e 100644
--- a/frappe/core/doctype/error_log/test_error_log.py
+++ b/frappe/core/doctype/error_log/test_error_log.py
@@ -1,10 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
+from unittest.mock import patch
+
 from ldap3.core.exceptions import LDAPException, LDAPInappropriateAuthenticationResult
 
 import frappe
 from frappe.tests.utils import FrappeTestCase
-from frappe.utils.error import _is_ldap_exception
+from frappe.utils.error import _is_ldap_exception, guess_exception_source
 
 # test_records = frappe.get_test_records('Error Log')
 
@@ -21,3 +23,44 @@ class TestErrorLog(FrappeTestCase):
 
 		for e in exc:
 			self.assertTrue(_is_ldap_exception(e()))
+
+
+_RAW_EXC = """
+   File "apps/frappe/frappe/model/document.py", line 1284, in runner
+     add_to_return_value(self, fn(self, *args, **kwargs))
+                               ^^^^^^^^^^^^^^^^^^^^^^^^^
+   File "apps/frappe/frappe/model/document.py", line 933, in fn
+     return method_object(*args, **kwargs)
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   File "apps/erpnext/erpnext/selling/doctype/sales_order/sales_order.py", line 58, in onload
+     raise Exception("what")
+ Exception: what
+"""
+
+_THROW_EXC = """
+   File "apps/frappe/frappe/model/document.py", line 933, in fn
+     return method_object(*args, **kwargs)
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   File "apps/erpnext/erpnext/selling/doctype/sales_order/sales_order.py", line 58, in onload
+     frappe.throw("what")
+   File "apps/frappe/frappe/__init__.py", line 550, in throw
+     msgprint(
+   File "apps/frappe/frappe/__init__.py", line 518, in msgprint
+     _raise_exception()
+   File "apps/frappe/frappe/__init__.py", line 467, in _raise_exception
+     raise raise_exception(msg)
+ frappe.exceptions.ValidationError: what
+"""
+
+TEST_EXCEPTIONS = {
+	"erpnext (app)": _RAW_EXC,
+	"erpnext (app)": _THROW_EXC,
+}
+
+
+class TestExceptionSourceGuessing(FrappeTestCase):
+	@patch.object(frappe, "get_installed_apps", return_value=["frappe", "erpnext", "3pa"])
+	def test_exc_source_guessing(self, _installed_apps):
+		for source, exc in TEST_EXCEPTIONS.items():
+			result = guess_exception_source(exc)
+			self.assertEqual(result, source)
diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js
index 6278d991c1..7e7e592669 100644
--- a/frappe/public/js/frappe/request.js
+++ b/frappe/public/js/frappe/request.js
@@ -604,7 +604,14 @@ frappe.request.report_error = function (xhr, request_opts) {
 
 		let parts = strip(exc).split("\n");
 
-		frappe.error_dialog.$body.html(parts[parts.length - 1]);
+		let dialog_html = parts[parts.length - 1];
+
+		if (data._exc_source) {
+			dialog_html += "<br>";
+			dialog_html += `Possible source of error: ${data._exc_source.bold()} `;
+		}
+
+		frappe.error_dialog.$body.html(dialog_html);
 		frappe.error_dialog.show();
 	}
 };
diff --git a/frappe/utils/error.py b/frappe/utils/error.py
index d44bdef47f..47902176ea 100644
--- a/frappe/utils/error.py
+++ b/frappe/utils/error.py
@@ -3,6 +3,9 @@
 
 import functools
 import inspect
+import re
+from collections import Counter
+from contextlib import suppress
 
 import frappe
 
@@ -126,3 +129,33 @@ def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None):
 		return wrapper_raise_error_on_no_output
 
 	return decorator_raise_error_on_no_output
+
+
+def guess_exception_source(exception: str) -> str | None:
+	"""Attempts to guess source of error based on traceback.
+
+	E.g.
+
+	- For unhandled exception last python file from apps folder is responsible.
+	- For frappe.throws the exception source is possibly present after skipping frappe.throw frames
+	- For server script the file name is `<serverscript>`
+
+	"""
+	with suppress(Exception):
+		installed_apps = frappe.get_installed_apps()
+		app_priority = {app: installed_apps.index(app) for app in installed_apps}
+
+		APP_NAME_REGEX = re.compile(r".*File.*apps/(?P<app_name>\w+)/\1/")
+		SERVER_SCRIPT_FRAME = re.compile(r".*<serverscript>")
+
+		apps = Counter()
+		for line in reversed(exception.splitlines()):
+			if SERVER_SCRIPT_FRAME.match(line):
+				return "Server Script"
+
+			if matches := APP_NAME_REGEX.match(line):
+				app_name = matches.group("app_name")
+				apps[app_name] += app_priority.get(app_name, 0)
+
+		if probably_source := apps.most_common(1):
+			return f"{probably_source[0][0]} (app)"
diff --git a/frappe/utils/response.py b/frappe/utils/response.py
index 0cdfd5b422..3ed3fe96d7 100644
--- a/frappe/utils/response.py
+++ b/frappe/utils/response.py
@@ -133,10 +133,14 @@ def as_binary():
 
 def make_logs(response=None):
 	"""make strings for msgprint and errprint"""
+	from frappe.utils.error import guess_exception_source
+
 	if not response:
 		response = frappe.local.response
 
 	if frappe.error_log:
+		if source := guess_exception_source(frappe.local.error_log and frappe.local.error_log[0]["exc"]):
+			response["_exc_source"] = source
 		response["exc"] = json.dumps([frappe.utils.cstr(d["exc"]) for d in frappe.local.error_log])
 
 	if frappe.local.message_log:

From 22a3a56d44a798d30e997c29e9f24e8f55445fc8 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 28 Jul 2023 11:45:56 +0530
Subject: [PATCH 32/60] chore: remove conflict-marker look-alike

This is annoying when trying to find git conflicts

[skip ci]
---
 frappe/database/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index a6254a0242..61cabc0478 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -314,7 +314,7 @@ class Database:
 
 		if frappe.conf.logging == 2:
 			_query = _query or str(mogrified_query)
-			frappe.log(f"<<<< query\n{_query}\n>>>>")
+			frappe.log(f"#### query\n{_query}\n####")
 
 		if unmogrified_query and is_query_type(
 			unmogrified_query, ("alter", "drop", "create", "truncate", "rename")

From ef51dde0526d967c0f873ef04efb272714492800 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 28 Jul 2023 13:01:46 +0530
Subject: [PATCH 33/60] perf: set lower priority for background processes
 (#21841)

---
 frappe/utils/background_jobs.py | 26 +++++++++++++++++++++++++-
 frappe/utils/scheduler.py       |  3 ++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py
index 19e2c57c0e..58cd133bd4 100755
--- a/frappe/utils/background_jobs.py
+++ b/frappe/utils/background_jobs.py
@@ -21,7 +21,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fi
 import frappe
 import frappe.monitor
 from frappe import _
-from frappe.utils import cstr, get_bench_id
+from frappe.utils import cint, cstr, get_bench_id
 from frappe.utils.commands import log
 from frappe.utils.deprecations import deprecation_warning
 from frappe.utils.redis_queue import RedisQueue
@@ -268,6 +268,8 @@ def start_worker(
 	if os.environ.get("CI"):
 		setup_loghandlers("ERROR")
 
+	set_niceness()
+
 	logging_level = "INFO"
 	if quiet:
 		logging_level = "WARNING"
@@ -305,6 +307,7 @@ def start_worker_pool(
 	if os.environ.get("CI"):
 		setup_loghandlers("ERROR")
 
+	set_niceness()
 	logging_level = "INFO"
 	if quiet:
 		logging_level = "WARNING"
@@ -519,3 +522,24 @@ def get_job(job_id: str) -> Job:
 		return Job.fetch(create_job_id(job_id), connection=get_redis_conn())
 	except NoSuchJobError:
 		return None
+
+
+BACKGROUND_PROCESS_NICENESS = 10
+
+
+def set_niceness():
+	"""Background processes should have slightly lower priority than web processes.
+
+	Calling this function increments the niceness of process by configured value or default.
+	Note: This function should be called only once in process' lifetime.
+	"""
+
+	conf = frappe.get_conf()
+	nice_increment = BACKGROUND_PROCESS_NICENESS
+
+	configured_niceness = conf.get("background_process_niceness")
+
+	if configured_niceness is not None:
+		nice_increment = cint(configured_niceness)
+
+	os.nice(nice_increment)
diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py
index 5916b018b4..8dd0b57e1f 100755
--- a/frappe/utils/scheduler.py
+++ b/frappe/utils/scheduler.py
@@ -16,7 +16,7 @@ from typing import NoReturn
 # imports - module imports
 import frappe
 from frappe.utils import cint, get_datetime, get_sites, now_datetime
-from frappe.utils.background_jobs import get_jobs
+from frappe.utils.background_jobs import set_niceness
 
 DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 
@@ -35,6 +35,7 @@ def start_scheduler() -> NoReturn:
 	Specify scheduler_interval in seconds in common_site_config.json"""
 
 	tick = cint(frappe.get_conf().scheduler_tick_interval) or 60
+	set_niceness()
 
 	while True:
 		time.sleep(tick)

From e31038d7f9ecad9ffec25f81dccdead5899198ce Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 28 Jul 2023 13:59:52 +0530
Subject: [PATCH 34/60] feat(sec): log IP addresses of login/logout activities
 (#21844)

---
 frappe/core/doctype/activity_log/activity_log.js   | 5 ++++-
 frappe/core/doctype/activity_log/activity_log.json | 8 +++++++-
 frappe/core/doctype/activity_log/activity_log.py   | 6 ++++++
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/activity_log/activity_log.js b/frappe/core/doctype/activity_log/activity_log.js
index 7df644a86a..39486dac6f 100644
--- a/frappe/core/doctype/activity_log/activity_log.js
+++ b/frappe/core/doctype/activity_log/activity_log.js
@@ -2,5 +2,8 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Activity Log", {
-	refresh: function () {},
+	refresh: function (frm) {
+		// Nothing in this form is supposed to be editable.
+		frm.disable_form();
+	},
 });
diff --git a/frappe/core/doctype/activity_log/activity_log.json b/frappe/core/doctype/activity_log/activity_log.json
index 910baceb5e..b272bab180 100644
--- a/frappe/core/doctype/activity_log/activity_log.json
+++ b/frappe/core/doctype/activity_log/activity_log.json
@@ -13,6 +13,7 @@
   "column_break_5",
   "additional_info",
   "communication_date",
+  "ip_address",
   "column_break_7",
   "operation",
   "status",
@@ -148,12 +149,17 @@
    "fieldtype": "Data",
    "in_list_view": 1,
    "label": "Full Name"
+  },
+  {
+   "fieldname": "ip_address",
+   "fieldtype": "Data",
+   "label": "IP Address"
   }
  ],
  "icon": "fa fa-comment",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-09-13 15:19:42.474114",
+ "modified": "2023-07-28 13:26:32.281278",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Activity Log",
diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py
index f7c21f6825..d58899f6cd 100644
--- a/frappe/core/doctype/activity_log/activity_log.py
+++ b/frappe/core/doctype/activity_log/activity_log.py
@@ -21,6 +21,7 @@ class ActivityLog(Document):
 		communication_date: DF.Datetime | None
 		content: DF.TextEditor | None
 		full_name: DF.Data | None
+		ip_address: DF.Data | None
 		link_doctype: DF.Link | None
 		link_name: DF.DynamicLink | None
 		operation: DF.Literal["", "Login", "Logout"]
@@ -40,6 +41,7 @@ class ActivityLog(Document):
 	def validate(self):
 		self.set_status()
 		set_timeline_doc(self)
+		self.set_ip_address()
 
 	def set_status(self):
 		if not self.is_new():
@@ -48,6 +50,10 @@ class ActivityLog(Document):
 		if self.reference_doctype and self.reference_name:
 			self.status = "Linked"
 
+	def set_ip_address(self):
+		if self.operation in ("Login", "Logout"):
+			self.ip_address = getattr(frappe.local, "request_ip")
+
 	@staticmethod
 	def clear_old_logs(days=None):
 		if not days:

From 1c5f5a38e9946503cd11976501ca0ee33cb2e6d5 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Fri, 28 Jul 2023 14:41:34 +0530
Subject: [PATCH 35/60] feat: added specific kanban board as quick link in
 workspace (backport #21721) (#21842)

* feat: added specific kanban board as quick link in workspace (#21721)

(cherry picked from commit 986b6024241036e68cfa4103d015e46a4c433b7b)

* fix: updated TYPE for workspace_shortcut

* chore: linter fix

---------

Co-authored-by: Vanessa Bualat <40702858+kkulloters@users.noreply.github.com>
Co-authored-by: Shariq Ansari <30859809+shariquerik@users.noreply.github.com>
---
 .../workspace_shortcut.json                   | 13 ++++++-
 .../workspace_shortcut/workspace_shortcut.py  |  5 ++-
 frappe/public/js/frappe/utils/utils.js        |  3 ++
 .../js/frappe/widgets/shortcut_widget.js      |  2 +
 .../public/js/frappe/widgets/widget_dialog.js | 38 ++++++++++++++++++-
 5 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
index 8832d9e1f4..e494aad152 100644
--- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
+++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
@@ -9,6 +9,7 @@
   "link_to",
   "url",
   "doc_view",
+  "kanban_board",
   "column_break_4",
   "label",
   "icon",
@@ -43,7 +44,7 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "DocType View",
-   "options": "\nList\nReport Builder\nDashboard\nTree\nNew\nCalendar"
+   "options": "\nList\nReport Builder\nDashboard\nTree\nNew\nCalendar\nKanban"
   },
   {
    "fieldname": "column_break_4",
@@ -103,12 +104,19 @@
    "in_list_view": 1,
    "label": "URL",
    "options": "URL"
+  },
+  {
+   "depends_on": "eval:doc.doc_view == \"Kanban\"",
+   "fieldname": "kanban_board",
+   "fieldtype": "Link",
+   "label": "Kanban Board",
+   "options": "Kanban Board"
   }
  ],
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-04-19 13:32:31.005443",
+ "modified": "2023-07-18 16:12:53.546430",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace Shortcut",
@@ -117,5 +125,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py
index 5b7cda15bf..9e908974fa 100644
--- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py
+++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py
@@ -15,9 +15,12 @@ class WorkspaceShortcut(Document):
 		from frappe.types import DF
 
 		color: DF.Color | None
-		doc_view: DF.Literal["", "List", "Report Builder", "Dashboard", "Tree", "New", "Calendar"]
+		doc_view: DF.Literal[
+			"", "List", "Report Builder", "Dashboard", "Tree", "New", "Calendar", "Kanban"
+		]
 		format: DF.Data | None
 		icon: DF.Data | None
+		kanban_board: DF.Link | None
 		label: DF.Data
 		link_to: DF.DynamicLink | None
 		parent: DF.Data
diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js
index e4cae790f2..b93b5ea8d5 100644
--- a/frappe/public/js/frappe/utils/utils.js
+++ b/frappe/public/js/frappe/utils/utils.js
@@ -1292,6 +1292,9 @@ Object.assign(frappe.utils, {
 							break;
 						case "Kanban":
 							route = `${doctype_slug}/view/kanban`;
+							if (item.kanban_board) {
+								route += `/${item.kanban_board}`;
+							}
 							break;
 						default:
 							route = doctype_slug;
diff --git a/frappe/public/js/frappe/widgets/shortcut_widget.js b/frappe/public/js/frappe/widgets/shortcut_widget.js
index 7d340b04e8..d0937f3b46 100644
--- a/frappe/public/js/frappe/widgets/shortcut_widget.js
+++ b/frappe/public/js/frappe/widgets/shortcut_widget.js
@@ -21,6 +21,7 @@ export default class ShortcutWidget extends Widget {
 			stats_filter: this.stats_filter,
 			type: this.type,
 			url: this.url,
+			kanban_board: this.kanban_board,
 		};
 	}
 
@@ -35,6 +36,7 @@ export default class ShortcutWidget extends Widget {
 				is_query_report: this.is_query_report,
 				doctype: this.ref_doctype,
 				doc_view: this.doc_view,
+				kanban_board: this.kanban_board,
 			});
 
 			let filters = frappe.utils.get_filter_from_json(this.stats_filter);
diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index 51138a7dc4..6818003bb6 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -384,7 +384,7 @@ class ShortcutDialog extends WidgetDialog {
 				onchange: () => {
 					const doctype = this.dialog.get_value("link_to");
 					if (doctype && this.dialog.get_value("type") == "DocType") {
-						frappe.model.with_doctype(doctype, () => {
+						frappe.model.with_doctype(doctype, async () => {
 							let meta = frappe.get_meta(doctype);
 
 							if (doctype && frappe.boot.single_types.includes(doctype)) {
@@ -398,6 +398,13 @@ class ShortcutDialog extends WidgetDialog {
 							if (meta.is_tree === "Tree") views.push("Tree");
 							if (frappe.boot.calendars.includes(doctype)) views.push("Calendar");
 
+							const response = await frappe.db.get_value(
+								"Kanban Board",
+								{ reference_doctype: doctype },
+								"name"
+							);
+							if (response?.message?.name) views.push("Kanban");
+
 							this.dialog.set_df_property("doc_view", "options", views.join("\n"));
 						});
 					} else {
@@ -429,11 +436,38 @@ class ShortcutDialog extends WidgetDialog {
 					if (this.dialog) {
 						let doctype = this.dialog.get_value("link_to");
 						let is_single = frappe.boot.single_types.includes(doctype);
-						return state.type == "DocType" && !is_single;
+						return doctype && state.type == "DocType" && !is_single;
 					}
 
 					return false;
 				},
+				onchange: () => {
+					if (this.dialog.get_value("doc_view") == "Kanban") {
+						this.dialog.fields_dict.kanban_board.get_query = () => {
+							return {
+								filters: {
+									reference_doctype: this.dialog.get_value("link_to"),
+								},
+							};
+						};
+					} else {
+						this.dialog.fields_dict.link_to.get_query = null;
+					}
+				},
+			},
+			{
+				fieldtype: "Link",
+				fieldname: "kanban_board",
+				label: "Kanban Board",
+				options: "Kanban Board",
+				depends_on: () => {
+					let doc_view = this.dialog?.get_value("doc_view");
+					return doc_view == "Kanban";
+				},
+				mandatory_depends_on: () => {
+					let doc_view = this.dialog?.get_value("doc_view");
+					return doc_view == "Kanban";
+				},
 			},
 			{
 				fieldtype: "Section Break",

From 7ee85758b1bd81cba651087d58efd7040ff4c928 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Fri, 28 Jul 2023 18:07:08 +0530
Subject: [PATCH 36/60] fix(console): dont commit when exception is raised and
 unconditionally check query (#21850)

* fix(console): unconditionally check query type

* fix(console): don't commit when exception occurs
---
 .../desk/doctype/system_console/system_console.py   |  1 +
 .../doctype/system_console/test_system_console.py   | 13 +++++++++++++
 frappe/utils/safe_exec.py                           |  3 +--
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py
index 76b67e32df..540936581a 100644
--- a/frappe/desk/doctype/system_console/system_console.py
+++ b/frappe/desk/doctype/system_console/system_console.py
@@ -33,6 +33,7 @@ class SystemConsole(Document):
 			elif self.type == "SQL":
 				self.output = frappe.as_json(read_sql(self.console, as_dict=1))
 		except Exception:
+			self.commit = False
 			self.output = frappe.get_traceback()
 
 		if self.commit:
diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py
index 2664f7c925..1be219509b 100644
--- a/frappe/desk/doctype/system_console/test_system_console.py
+++ b/frappe/desk/doctype/system_console/test_system_console.py
@@ -16,3 +16,16 @@ class TestSystemConsole(FrappeTestCase):
 		system_console.run()
 
 		self.assertEqual(system_console.output, "Core")
+
+	def test_system_console_sql(self):
+		system_console = frappe.get_doc("System Console")
+		system_console.type = "SQL"
+		system_console.console = "select 1"
+		system_console.run()
+
+		self.assertEqual(frappe.parse_json(system_console.output), [{"1": 1}])
+
+		system_console.console = "update `tabDocType` set is_virtual = 1 where name = 'xyz'"
+		system_console.run()
+
+		self.assertIn("PermissionError", system_console.output)
diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py
index 50dae80f8f..c0bfd3cbbb 100644
--- a/frappe/utils/safe_exec.py
+++ b/frappe/utils/safe_exec.py
@@ -330,8 +330,7 @@ def get_hooks(hook=None, default=None, app_name=None):
 def read_sql(query, *args, **kwargs):
 	"""a wrapper for frappe.db.sql to allow reads"""
 	query = str(query)
-	if frappe.flags.in_safe_exec:
-		check_safe_sql_query(query)
+	check_safe_sql_query(query)
 	return frappe.db.sql(query, *args, **kwargs)
 
 

From 6b7f815a755cdafb42048b66dfb3a3c4df6381d9 Mon Sep 17 00:00:00 2001
From: Maharshi Patel <39730881+maharshivpatel@users.noreply.github.com>
Date: Fri, 28 Jul 2023 18:13:22 +0530
Subject: [PATCH 37/60] fix: guess_country backward timezone issue for india.
 (#21849)

* fix: guess_country backward timezone issue for india.

In some cases ( mac ) where Asia/Calcutta is used as timezone. guessing didn't work.

* refactor: make tz backward compatbility extensible

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/desk/page/setup_wizard/setup_wizard.js | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js
index decf540097..36658fe492 100644
--- a/frappe/desk/page/setup_wizard/setup_wizard.js
+++ b/frappe/desk/page/setup_wizard/setup_wizard.js
@@ -638,9 +638,15 @@ frappe.setup.utils = {
 	},
 };
 
+// https://github.com/eggert/tz/blob/main/backward add more if required.
+const TZ_BACKWARD_COMPATBILITY_MAP = {
+	"Asia/Calcutta": "Asia/Kolkata",
+};
+
 function guess_country(country_info) {
 	try {
-		const system_timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+		let system_timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+		system_timezone = TZ_BACKWARD_COMPATBILITY_MAP[system_timezone] || system_timezone;
 
 		for (let [country, info] of Object.entries(country_info)) {
 			let possible_timezones = (info.timezones || []).filter((t) => t == system_timezone);

From 90f6b44f485e85c1ab27f851594d0410e0b1088c Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Fri, 28 Jul 2023 19:01:11 +0530
Subject: [PATCH 38/60] fix: added welcome workspace if user has desk acces but
 does not have module access

basically if user doesn't have access to any public workspaces

also made welcome workspace hidden if he/she has even one workspace
---
 .../welcome_workspace/welcome_workspace.json  | 28 +++++++++++++++++++
 frappe/desk/desktop.py                        |  6 +++-
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 frappe/core/workspace/welcome_workspace/welcome_workspace.json

diff --git a/frappe/core/workspace/welcome_workspace/welcome_workspace.json b/frappe/core/workspace/welcome_workspace/welcome_workspace.json
new file mode 100644
index 0000000000..426dcb8a4e
--- /dev/null
+++ b/frappe/core/workspace/welcome_workspace/welcome_workspace.json
@@ -0,0 +1,28 @@
+{
+ "charts": [],
+ "content": "[{\"id\":\"2eyXSHwMTE\",\"type\":\"header\",\"data\":{\"text\":\"<span class=\\\"h2\\\">Hi,</span>\",\"col\":12}},{\"id\":\"ZusKvFOXgu\",\"type\":\"paragraph\",\"data\":{\"text\":\"I guess you do not have access to any standard workspaces, you can create your own by clicking on <b>Create Workspace</b> button on top right\",\"col\":12}}]",
+ "creation": "2023-07-28 17:14:28.608321",
+ "custom_blocks": [],
+ "docstatus": 0,
+ "doctype": "Workspace",
+ "for_user": "",
+ "hide_custom": 0,
+ "icon": "image-view",
+ "idx": 0,
+ "is_hidden": 0,
+ "label": "Welcome Workspace",
+ "links": [],
+ "modified": "2023-07-28 18:45:23.125415",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Welcome Workspace",
+ "number_cards": [],
+ "owner": "Administrator",
+ "parent_page": "",
+ "public": 1,
+ "quick_lists": [],
+ "roles": [],
+ "sequence_id": 22.0,
+ "shortcuts": [],
+ "title": "Welcome Workspace"
+}
\ No newline at end of file
diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py
index cf9f223d2a..141ac7d013 100644
--- a/frappe/desk/desktop.py
+++ b/frappe/desk/desktop.py
@@ -453,7 +453,7 @@ def get_workspace_sidebar_items():
 		try:
 			workspace = Workspace(page, True)
 			if has_access or workspace.is_permitted():
-				if page.public and (has_access or not page.is_hidden):
+				if page.public and (has_access or not page.is_hidden) and page.title != "Welcome Workspace":
 					pages.append(page)
 				elif page.for_user == frappe.session.user:
 					private_pages.append(page)
@@ -463,6 +463,10 @@ def get_workspace_sidebar_items():
 	if private_pages:
 		pages.extend(private_pages)
 
+	if len(pages) == 0:
+		pages = [frappe.get_doc("Workspace", "Welcome Workspace").as_dict()]
+		pages[0]["label"] = _("Welcome Workspace")
+
 	return {"pages": pages, "has_access": has_access}
 
 

From 24a5b386345ad0f8b158bf16a63ab9c21d0e16c8 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Fri, 28 Jul 2023 19:01:36 +0530
Subject: [PATCH 39/60] fix: better redirect logic for default workspace after
 login

---
 frappe/public/js/frappe/router.js | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js
index 9ad917a57a..c4ee4d2659 100644
--- a/frappe/public/js/frappe/router.js
+++ b/frappe/public/js/frappe/router.js
@@ -445,11 +445,17 @@ frappe.router = {
 			}
 		}).join("/");
 		let private_home = frappe.workspaces[`home-${frappe.user.name.toLowerCase()}`];
-		let default_page = private_home
-			? "private/home"
-			: frappe.workspaces["home"]
-			? "home"
-			: Object.keys(frappe.workspaces)[0];
+
+		let workspace_name = private_home || frappe.workspaces["home"] ? "home" : "";
+		let is_private = !!private_home;
+		let first_workspace = Object.keys(frappe.workspaces)[0];
+
+		if (!workspace_name && first_workspace) {
+			workspace_name = frappe.workspaces[first_workspace].title;
+			is_private = !frappe.workspaces[first_workspace].public;
+		}
+
+		let default_page = (is_private ? "private/" : "") + frappe.router.slug(workspace_name);
 		return "/app/" + (path_string || default_page);
 	},
 

From a227944cfc36439e9b4ec418866e30dad2e79bf6 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Fri, 28 Jul 2023 19:01:54 +0530
Subject: [PATCH 40/60] fix: default page logic in workspace

---
 frappe/public/js/frappe/views/workspace/workspace.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js
index 2bf717c1e7..9b135338ca 100644
--- a/frappe/public/js/frappe/views/workspace/workspace.js
+++ b/frappe/public/js/frappe/views/workspace/workspace.js
@@ -326,7 +326,7 @@ frappe.views.Workspace = class Workspace {
 				public: localStorage.is_current_page_public == "true",
 			};
 		} else if (Object.keys(this.all_pages).length !== 0) {
-			default_page = { name: this.all_pages[0].title, public: true };
+			default_page = { name: this.all_pages[0].title, public: this.all_pages[0].public };
 		} else {
 			default_page = { name: "Build", public: true };
 		}
@@ -344,10 +344,11 @@ frappe.views.Workspace = class Workspace {
 			`).appendTo(this.body);
 		}
 
-		if (this.all_pages) {
+		if (this.all_pages.length) {
 			this.create_page_skeleton();
 
-			let pages = page.public ? this.public_pages : this.private_pages;
+			let pages =
+				page.public && this.public_pages.length ? this.public_pages : this.private_pages;
 			let current_page = pages.filter((p) => p.title == page.name)[0];
 			this.content = current_page && JSON.parse(current_page.content);
 

From e4c45dd719cf9fe3ef754e96746cf87ea8fd0391 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Fri, 28 Jul 2023 19:59:58 +0530
Subject: [PATCH 41/60] fix: after creating workspace and reloading it gives
 error

also if workspace does not exist it gives proper error and redirect to last opened workspaces
---
 frappe/public/js/frappe/router.js                    | 12 +++++++++++-
 frappe/public/js/frappe/views/workspace/workspace.js |  1 +
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js
index c4ee4d2659..2631c6374e 100644
--- a/frappe/public/js/frappe/router.js
+++ b/frappe/public/js/frappe/router.js
@@ -177,8 +177,18 @@ frappe.router = {
 		if (frappe.workspaces[route[0]]) {
 			// public workspace
 			route = ["Workspaces", frappe.workspaces[route[0]].title];
-		} else if (route[0] == "private" && frappe.workspaces[private_workspace]) {
+		} else if (route[0] == "private") {
 			// private workspace
+			if (!frappe.workspaces[private_workspace] && localStorage.new_workspace) {
+				let new_workspace = JSON.parse(localStorage.new_workspace);
+				if (frappe.router.slug(new_workspace.title) === route[1]) {
+					frappe.workspaces[private_workspace] = new_workspace;
+				}
+			}
+			if (!frappe.workspaces[private_workspace]) {
+				frappe.msgprint(__("Workspace <b>{0}</b> does not exist", [route[1]]));
+				return ["Workspaces"];
+			}
 			route = ["Workspaces", "private", frappe.workspaces[private_workspace].title];
 		} else if (this.routes[route[0]]) {
 			// route
diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js
index 9b135338ca..9dc5c07a95 100644
--- a/frappe/public/js/frappe/views/workspace/workspace.js
+++ b/frappe/public/js/frappe/views/workspace/workspace.js
@@ -1210,6 +1210,7 @@ frappe.views.Workspace = class Workspace {
 
 						this.make_sidebar();
 						this.show_sidebar_actions();
+						localStorage.setItem("new_workspace", JSON.stringify(new_page));
 					});
 			},
 		});

From 8a8716f21c2cf38fbe80136e142c2bcceafc1b95 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Fri, 28 Jul 2023 20:16:29 +0530
Subject: [PATCH 42/60] chore: updated the welcome message

---
 .../core/workspace/welcome_workspace/welcome_workspace.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/core/workspace/welcome_workspace/welcome_workspace.json b/frappe/core/workspace/welcome_workspace/welcome_workspace.json
index 426dcb8a4e..1b45434990 100644
--- a/frappe/core/workspace/welcome_workspace/welcome_workspace.json
+++ b/frappe/core/workspace/welcome_workspace/welcome_workspace.json
@@ -1,6 +1,6 @@
 {
  "charts": [],
- "content": "[{\"id\":\"2eyXSHwMTE\",\"type\":\"header\",\"data\":{\"text\":\"<span class=\\\"h2\\\">Hi,</span>\",\"col\":12}},{\"id\":\"ZusKvFOXgu\",\"type\":\"paragraph\",\"data\":{\"text\":\"I guess you do not have access to any standard workspaces, you can create your own by clicking on <b>Create Workspace</b> button on top right\",\"col\":12}}]",
+ "content": "[{\"id\":\"2eyXSHwMTE\",\"type\":\"header\",\"data\":{\"text\":\"<span class=\\\"h2\\\">Hi,</span>\",\"col\":12}},{\"id\":\"ZusKvFOXgu\",\"type\":\"paragraph\",\"data\":{\"text\":\"I guess you don't have access to any workspace yet, but you can create one just for yourself. Click on the <b>Create Workspace</b> button to create one.<br>\",\"col\":12}}]",
  "creation": "2023-07-28 17:14:28.608321",
  "custom_blocks": [],
  "docstatus": 0,
@@ -8,11 +8,11 @@
  "for_user": "",
  "hide_custom": 0,
  "icon": "image-view",
- "idx": 0,
+ "idx": 1,
  "is_hidden": 0,
  "label": "Welcome Workspace",
  "links": [],
- "modified": "2023-07-28 18:45:23.125415",
+ "modified": "2023-07-28 20:15:32.222029",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Welcome Workspace",

From d7aea1773fae50b35f30394eef0b0a88300e8f21 Mon Sep 17 00:00:00 2001
From: Corentin Forler <8860073-cforler_dokos@users.noreply.gitlab.com>
Date: Fri, 28 Jul 2023 17:52:53 +0200
Subject: [PATCH 43/60] fix(web_form): Allow Jinja extension of header buttons

---
 frappe/website/doctype/web_form/templates/web_form.html | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/frappe/website/doctype/web_form/templates/web_form.html b/frappe/website/doctype/web_form/templates/web_form.html
index cbc4c8ac0a..6bda7f8e49 100644
--- a/frappe/website/doctype/web_form/templates/web_form.html
+++ b/frappe/website/doctype/web_form/templates/web_form.html
@@ -14,6 +14,7 @@
 {% endblock %}
 
 {% macro header_buttons() %}
+	{% block header_buttons %}
 	{% if allow_edit and in_view_mode %}
 		<!-- edit button -->
 		<a href="/{{ route }}/{{ doc_name }}/edit" class="edit-button btn btn-default btn-sm">{{ _("Edit Response", null, "Button in web form") }}</a>
@@ -26,9 +27,11 @@
 			<svg class="icon icon-sm"><use href="#icon-printer"></use></svg>
 		</a>
 	{% endif %}
+	{% endblock %}
 {% endmacro %}
 
 {% macro action_buttons() %}
+	{% block actions_buttons %}
 	<div class="left-area"></div>
 	<div class="center-area paging"></div>
 	<div class="right-area">
@@ -41,6 +44,7 @@
 			<button type="submit" class="submit-btn btn btn-primary btn-sm ml-2">{{ button_label or _("Submit", null, "Button in web form") }}</button>
 		{% endif %}
 	</div>
+	{% endblock %}
 {% endmacro %}
 
 {% block page_content %}

From 120b9876616eb4be4f9d710e701b8ac201ad64de Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Fri, 28 Jul 2023 23:04:05 +0530
Subject: [PATCH 44/60] fix: send public with title in frappe.workspaces

---
 frappe/public/js/frappe/views/workspace/workspace.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js
index 9dc5c07a95..55984a1d43 100644
--- a/frappe/public/js/frappe/views/workspace/workspace.js
+++ b/frappe/public/js/frappe/views/workspace/workspace.js
@@ -65,7 +65,10 @@ frappe.views.Workspace = class Workspace {
 		if (this.all_pages) {
 			frappe.workspaces = {};
 			for (let page of this.all_pages) {
-				frappe.workspaces[frappe.router.slug(page.name)] = { title: page.title };
+				frappe.workspaces[frappe.router.slug(page.name)] = {
+					title: page.title,
+					public: page.public,
+				};
 			}
 			this.make_sidebar();
 			reload && this.show();

From f5a2eef755afd78b2e135ff1a43c699693ef8938 Mon Sep 17 00:00:00 2001
From: Shariq Ansari <sharique.rik@gmail.com>
Date: Fri, 28 Jul 2023 23:16:38 +0530
Subject: [PATCH 45/60] test: fixed failing date control UI test

---
 cypress/integration/control_color.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cypress/integration/control_color.js b/cypress/integration/control_color.js
index e97dbe0f06..aa3a45eed8 100644
--- a/cypress/integration/control_color.js
+++ b/cypress/integration/control_color.js
@@ -26,7 +26,7 @@ context("Control Color", () => {
 
 		//Checking if the css attribute is correct
 		cy.get(".color-map").should("have.css", "color", "rgb(79, 157, 217)");
-		cy.get(".hue-map").should("have.css", "color", "rgb(0, 144, 255)");
+		cy.get(".hue-map").should("have.css", "color", "rgb(0, 145, 255)");
 
 		//Checking if the correct color is being selected
 		cy.get("@dialog").then((dialog) => {

From a2167d4f9440e03d25e9626cdbf15c5e5d6875fb Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Sat, 29 Jul 2023 13:47:44 +0530
Subject: [PATCH 46/60] perf: index email queue table

---
 .../doctype/email_queue_recipient/email_queue_recipient.py   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
index 43b85c9f22..2d0c5678f7 100644
--- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
+++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
@@ -33,3 +33,8 @@ class EmailQueueRecipient(Document):
 		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
 		if commit:
 			frappe.db.commit()
+
+
+def on_doctype_update():
+	"""Index required for log clearing, modified is not indexed on child table by default"""
+	frappe.db.add_index("Email Queue Recipient", ["modified"])

From 657ec80e3a648bb7171a407872f6d091ae7c0c3b Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Sat, 29 Jul 2023 18:06:02 +0530
Subject: [PATCH 47/60] chore: skip console test on postgres

[skip ci]
---
 frappe/desk/doctype/system_console/test_system_console.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py
index 1be219509b..ade8704813 100644
--- a/frappe/desk/doctype/system_console/test_system_console.py
+++ b/frappe/desk/doctype/system_console/test_system_console.py
@@ -20,10 +20,10 @@ class TestSystemConsole(FrappeTestCase):
 	def test_system_console_sql(self):
 		system_console = frappe.get_doc("System Console")
 		system_console.type = "SQL"
-		system_console.console = "select 1"
+		system_console.console = "select 'test'"
 		system_console.run()
 
-		self.assertEqual(frappe.parse_json(system_console.output), [{"1": 1}])
+		self.assertIn("test", system_console.output)
 
 		system_console.console = "update `tabDocType` set is_virtual = 1 where name = 'xyz'"
 		system_console.run()

From ec5f8fcfa10ca9a90186208d52416d0c61dcedd8 Mon Sep 17 00:00:00 2001
From: HarryPaulo <paulo_fabris@hotmail.com>
Date: Sat, 29 Jul 2023 09:40:42 -0300
Subject: [PATCH 48/60] fix: Parameter {function} get translated and format
 doesn't work (#21859)

---
 frappe/core/doctype/report/report.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 920fc3f806..64f125eae2 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -411,9 +411,7 @@ def get_group_by_column_label(args, meta):
 	else:
 		sql_fn_map = {"avg": "Average", "sum": "Sum"}
 		aggregate_on_label = meta.get_label(args.aggregate_on)
-		label = _("{function} of {fieldlabel}").format(
-			function=sql_fn_map[args.aggregate_function], fieldlabel=aggregate_on_label
-		)
+		label = _("{0} of {1}").format(_(sql_fn_map[args.aggregate_function]), _(aggregate_on_label))
 	return label
 
 

From 84188587fa330a05b4c717a25cf1474e55757dd4 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Sat, 29 Jul 2023 20:33:55 +0530
Subject: [PATCH 49/60] fix: clean up socketio dynamic namespaces

While serving many sites it's best to close down namespace when last
connection is closed.

refer https://github.com/socketio/socket.io/pull/4602
---
 realtime/index.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/realtime/index.js b/realtime/index.js
index c60ec66330..2fc873b863 100644
--- a/realtime/index.js
+++ b/realtime/index.js
@@ -9,6 +9,7 @@ let io = new Server({
 		origin: true,
 		credentials: true,
 	},
+	cleanupEmptyChildNamespaces: true,
 });
 
 // Multitenancy implementation.

From c66d7448246e9578b30c5b59eabd0624b21aa864 Mon Sep 17 00:00:00 2001
From: Dirk van der Laarse <dirk@laarse.co.za>
Date: Sun, 30 Jul 2023 15:36:55 +0000
Subject: [PATCH 50/60] fix: field order and change label to title case

---
 frappe/core/doctype/system_settings/system_settings.json | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json
index 5d88a3e466..95dc1af924 100644
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@@ -33,6 +33,7 @@
   "apply_strict_user_permissions",
   "column_break_21",
   "allow_guests_to_upload_files",
+  "force_web_capture_mode_for_uploads",
   "security",
   "session_expiry",
   "document_share_key_expiry",
@@ -569,13 +570,13 @@
    "description": "When uploading files, force the use of the web-based image capture. If this is unchecked, the default behavior is to use the mobile native camera when use from a mobile is detected.",
    "fieldname": "force_web_capture_mode_for_uploads",
    "fieldtype": "Check",
-   "label": "Force web capture mode for uploads"
+   "label": "Force Web Capture Mode for Uploads"
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2023-06-14 11:19:33.429196",
+ "modified": "2023-07-30 17:34:08.292152",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

From 6645589cdf7682c327b4015e057def2bb397ceff Mon Sep 17 00:00:00 2001
From: Dirk van der Laarse <dirk@laarse.co.za>
Date: Sun, 30 Jul 2023 15:48:03 +0000
Subject: [PATCH 51/60] chore: linting fixes

---
 cypress/integration/control_attach.js | 30 +++++++++++++--------------
 frappe/public/js/frappe/ui/capture.js | 11 ++++------
 2 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/cypress/integration/control_attach.js b/cypress/integration/control_attach.js
index d6086c9ed1..6714f6c24e 100644
--- a/cypress/integration/control_attach.js
+++ b/cypress/integration/control_attach.js
@@ -95,17 +95,17 @@ context("Attach Control", () => {
 
 	it('Checking that "Camera" button in the "Attach" fieldtype does show if camera is available', () => {
 		//Navigating to the new form for the newly created doctype
-		let doctype = "Test Attach Control"
+		let doctype = "Test Attach Control";
 		let dt_in_route = doctype.toLowerCase().replace(/ /g, "-");
 		cy.visit(`/app/${dt_in_route}/new`, {
-			onBeforeLoad (win) {
-			  // Mock "window.navigator.mediaDevices" property
-			  // to return mock mediaDevices object
-			  win.navigator.mediaDevices = {
-				ondevicechange: null
-			  }
-			}
-		  });
+			onBeforeLoad(win) {
+				// Mock "window.navigator.mediaDevices" property
+				// to return mock mediaDevices object
+				win.navigator.mediaDevices = {
+					ondevicechange: null,
+				};
+			},
+		});
 		cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`);
 		cy.get("body").should("have.attr", "data-ajax-state", "complete");
 
@@ -114,19 +114,18 @@ context("Attach Control", () => {
 
 		//Clicking on "Camera" button
 		cy.findByRole("button", { name: "Camera" }).should("exist");
-
 	});
 
 	it('Checking that "Camera" button in the "Attach" fieldtype does not show if no camera is available', () => {
 		//Navigating to the new form for the newly created doctype
-		let doctype = "Test Attach Control"
+		let doctype = "Test Attach Control";
 		let dt_in_route = doctype.toLowerCase().replace(/ /g, "-");
 		cy.visit(`/app/${dt_in_route}/new`, {
-			onBeforeLoad (win) {
+			onBeforeLoad(win) {
 				// Delete "window.navigator.mediaDevices" property
-				delete win.navigator.mediaDevices
-			}
-			});
+				delete win.navigator.mediaDevices;
+			},
+		});
 		cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`);
 		cy.get("body").should("have.attr", "data-ajax-state", "complete");
 
@@ -135,6 +134,5 @@ context("Attach Control", () => {
 
 		//Clicking on "Camera" button
 		cy.findByRole("button", { name: "Camera" }).should("not.exist");
-			
 	});
 });
diff --git a/frappe/public/js/frappe/ui/capture.js b/frappe/public/js/frappe/ui/capture.js
index 8309ffeeb8..dc4804be52 100644
--- a/frappe/public/js/frappe/ui/capture.js
+++ b/frappe/public/js/frappe/ui/capture.js
@@ -78,13 +78,10 @@ frappe.ui.Capture = class {
 
 		if (cint(frappe.boot.sysdefaults.force_web_capture_mode_for_uploads)) {
 			this.show_for_desktop();
-		}
-		else {
-			if (frappe.is_mobile()) {
-				this.show_for_mobile();
-			} else {
-				this.show_for_desktop();
-			}
+		} else if (frappe.is_mobile()) {
+			this.show_for_mobile();
+		} else {
+			this.show_for_desktop();
 		}
 	}
 

From d08ee0284d6f768766d3c64b45a296c3f6d95f48 Mon Sep 17 00:00:00 2001
From: niraj gautam <nirajgautam196@gmail.com>
Date: Mon, 31 Jul 2023 11:39:04 +0530
Subject: [PATCH 52/60] fix: Delete module folder on Module Def deletion
 (#21857)

* fix: Delete module folder on Module Def deletion

* fix: Check if folder exists before removal

* fix: Make use of frappe function to delete folder

* fix: Indentation

* fix: Update linting
---
 frappe/core/doctype/module_def/module_def.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/core/doctype/module_def/module_def.py b/frappe/core/doctype/module_def/module_def.py
index f0cbda34cd..9acc9953c6 100644
--- a/frappe/core/doctype/module_def/module_def.py
+++ b/frappe/core/doctype/module_def/module_def.py
@@ -6,6 +6,7 @@ import os
 
 import frappe
 from frappe.model.document import Document
+from frappe.modules.export_file import delete_folder
 
 
 class ModuleDef(Document):
@@ -22,6 +23,7 @@ class ModuleDef(Document):
 		module_name: DF.Data
 		package: DF.Link | None
 		restrict_to_domain: DF.Link | None
+
 	# end: auto-generated types
 	def on_update(self):
 		"""If in `developer_mode`, create folder for module and
@@ -64,6 +66,7 @@ class ModuleDef(Document):
 
 		modules = None
 		if frappe.local.module_app.get(frappe.scrub(self.name)):
+			delete_folder(self.module_name, "Module Def", self.name)
 			with open(frappe.get_app_path(self.app_name, "modules.txt")) as f:
 				content = f.read()
 				if self.name in content.splitlines():

From 97eefdaa44faaa935385f43f8b20bd591bdd2271 Mon Sep 17 00:00:00 2001
From: Sagar Vora <sagar@resilient.tech>
Date: Mon, 31 Jul 2023 06:18:54 +0000
Subject: [PATCH 53/60] perf: dont reload doc when already saving (#21869)

---
 frappe/public/js/frappe/model/model.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js
index 0802d8b34c..a4a12af54e 100644
--- a/frappe/public/js/frappe/model/model.js
+++ b/frappe/public/js/frappe/model/model.js
@@ -149,10 +149,10 @@ $.extend(frappe.model, {
 					cur_frm.doc.doctype === doc.doctype &&
 					cur_frm.doc.name === doc.name
 				) {
-					if (data.modified !== cur_frm.doc.modified) {
+					if (data.modified !== cur_frm.doc.modified && !frappe.ui.form.is_saving) {
 						if (!cur_frm.is_dirty()) {
 							cur_frm.reload_doc();
-						} else if (!frappe.ui.form.is_saving) {
+						} else {
 							doc.__needs_refresh = true;
 							cur_frm.show_conflict_message();
 						}

From 3471685eaa046b3db805984f425be13fd9022f92 Mon Sep 17 00:00:00 2001
From: David Arnold <david.arnold@iohk.io>
Date: Mon, 31 Jul 2023 01:20:09 -0500
Subject: [PATCH 54/60] feat(ops): add environment variable to set bench root /
 workdir (#21872)

---
 frappe/utils/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py
index dbe6c5b925..3f05734c41 100644
--- a/frappe/utils/__init__.py
+++ b/frappe/utils/__init__.py
@@ -509,7 +509,9 @@ def get_files_path(*path, **kwargs):
 
 
 def get_bench_path():
-	return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), "..", "..", ".."))
+	return os.environ.get("FRAPPE_BENCH_ROOT") or os.path.realpath(
+		os.path.join(os.path.dirname(frappe.__file__), "..", "..", "..")
+	)
 
 
 def get_bench_id():

From fddd3b24f7ab56bd9e2309cf651f2f02a164fa60 Mon Sep 17 00:00:00 2001
From: David Arnold <david.arnold@iohk.io>
Date: Mon, 31 Jul 2023 02:16:23 -0500
Subject: [PATCH 55/60] test: skip tests on RO app paths which require write
 access (#21878)

---
 frappe/core/doctype/doctype/test_doctype.py      | 14 ++++++++++++++
 .../doctype/print_format/test_print_format.py    |  4 ++++
 frappe/tests/test_boilerplate.py                 |  9 +++++++++
 frappe/tests/test_modules.py                     | 16 ++++++++++++++++
 4 files changed, 43 insertions(+)

diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py
index c819663962..40c55c594e 100644
--- a/frappe/core/doctype/doctype/test_doctype.py
+++ b/frappe/core/doctype/doctype/test_doctype.py
@@ -1,7 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
+import os
 import random
 import string
+import unittest
 from unittest.mock import patch
 
 import frappe
@@ -172,6 +174,9 @@ class TestDocType(FrappeTestCase):
 				if condition:
 					self.assertFalse(re.match(pattern, condition))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_sync_field_order(self):
 		import os
 
@@ -648,6 +653,9 @@ class TestDocType(FrappeTestCase):
 	def test_no_delete_doc(self):
 		self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address")
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	@patch.dict(frappe.conf, {"developer_mode": 1})
 	def test_export_types(self):
 		"""Export python types."""
@@ -686,6 +694,9 @@ class TestDocType(FrappeTestCase):
 		doctype.delete()
 		frappe.db.commit()
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	@patch.dict(frappe.conf, {"developer_mode": 1})
 	def test_custom_field_deletion(self):
 		"""Custom child tables whose doctype doesn't exist should be auto deleted."""
@@ -698,6 +709,9 @@ class TestDocType(FrappeTestCase):
 		frappe.delete_doc("DocType", child)
 		self.assertFalse(frappe.get_meta(doctype).get_field(field))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	@patch.dict(frappe.conf, {"developer_mode": 1})
 	def test_delete_doctype_with_customization(self):
 		from frappe.custom.doctype.property_setter.property_setter import make_property_setter
diff --git a/frappe/printing/doctype/print_format/test_print_format.py b/frappe/printing/doctype/print_format/test_print_format.py
index 7caa5c6102..c54e421861 100644
--- a/frappe/printing/doctype/print_format/test_print_format.py
+++ b/frappe/printing/doctype/print_format/test_print_format.py
@@ -2,6 +2,7 @@
 # License: MIT. See LICENSE
 import os
 import re
+import unittest
 from typing import TYPE_CHECKING
 
 import frappe
@@ -36,6 +37,9 @@ class TestPrintFormat(FrappeTestCase):
 		print_html = self.test_print_user("Classic")
 		self.assertTrue("/* classic format: for-test */" in print_html)
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_export_doc(self):
 		doc: "PrintFormat" = frappe.get_doc("Print Format", test_records[0]["name"])
 
diff --git a/frappe/tests/test_boilerplate.py b/frappe/tests/test_boilerplate.py
index 4a405de2f8..9a2b774977 100644
--- a/frappe/tests/test_boilerplate.py
+++ b/frappe/tests/test_boilerplate.py
@@ -111,6 +111,9 @@ class TestBoilerPlate(unittest.TestCase):
 	def test_valid_ci_yaml(self):
 		yaml.safe_load(github_workflow_template.format(**self.default_hooks))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_create_app(self):
 		app_name = "test_app"
 
@@ -142,6 +145,9 @@ class TestBoilerPlate(unittest.TestCase):
 
 		self.assertEqual(parse_as_configfile(patches_file), [])
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_create_app_without_git_init(self):
 		app_name = "test_app_no_git"
 
@@ -192,6 +198,9 @@ class TestBoilerPlate(unittest.TestCase):
 				except Exception as e:
 					self.fail(f"Can't parse python file in new app: {python_file}\n" + str(e))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_new_patch_util(self):
 		user_inputs = {
 			"app_name": "frappe",
diff --git a/frappe/tests/test_modules.py b/frappe/tests/test_modules.py
index d355577e99..a19b45691f 100644
--- a/frappe/tests/test_modules.py
+++ b/frappe/tests/test_modules.py
@@ -1,5 +1,6 @@
 import os
 import shutil
+import unittest
 
 import frappe
 from frappe import scrub
@@ -82,6 +83,9 @@ class TestUtils(FrappeTestCase):
 		doc = frappe.get_last_doc("DocType")
 		self.assertIsNone(export_module_json(doc=doc, is_standard=True, module=doc.module))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_export_module_json(self):
 		doc = frappe.get_last_doc("DocType", {"issingle": 0, "custom": 0})
 		export_doc_path = os.path.join(
@@ -107,12 +111,18 @@ class TestUtils(FrappeTestCase):
 
 		self.assertTrue(last_modified_after > last_modified_before)
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_export_customizations(self):
 		file_path = export_customizations(module="Custom", doctype="Note")
 		self.addCleanup(delete_file, path=file_path)
 		self.assertTrue(file_path.endswith("/custom/custom/note.json"))
 		self.assertTrue(os.path.exists(file_path))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_sync_customizations(self):
 		custom_field = frappe.get_doc(
 			"Custom Field", {"dt": "Note", "fieldname": "test_export_customizations_field"}
@@ -157,6 +167,9 @@ class TestUtils(FrappeTestCase):
 		)
 		self.assertTrue(frappe.db.get_value("DocType", "Note", "migration_hash"))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_export_doc(self):
 		exported_doc_path = frappe.get_app_path(
 			"frappe", "desk", "note", self.note.name, f"{self.note.name}.json"
@@ -166,6 +179,9 @@ class TestUtils(FrappeTestCase):
 		self.addCleanup(delete_path, path=folder_path)
 		self.assertTrue(os.path.exists(exported_doc_path))
 
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
 	def test_make_boilerplate(self):
 		scrubbed = frappe.scrub(self.doctype.name)
 		self.assertFalse(

From ff6439af3d2dc288cbfef168a31efa9cab937934 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Mon, 31 Jul 2023 14:44:52 +0530
Subject: [PATCH 56/60] fix: underlined onboarding steps

---
 frappe/public/scss/desk/desktop.scss | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss
index 2d13b22883..44d3219f7d 100644
--- a/frappe/public/scss/desk/desktop.scss
+++ b/frappe/public/scss/desk/desktop.scss
@@ -415,6 +415,7 @@ body {
 				padding: 8px;
 				font-size: var(--text-md);
 				max-width: 350px;
+				text-decoration: none;
 
 				&.pending {
 					.step-index.step-pending {

From b41d4905c043373e8fc40b5d32e01b6f91091305 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Mon, 31 Jul 2023 14:58:09 +0530
Subject: [PATCH 57/60] fix: onboarding widget contrast

---
 frappe/public/scss/desk/desktop.scss | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss
index 44d3219f7d..46ce5528d0 100644
--- a/frappe/public/scss/desk/desktop.scss
+++ b/frappe/public/scss/desk/desktop.scss
@@ -341,7 +341,7 @@ body {
 	&.onboarding-widget-box {
 		margin-bottom: var(--margin-2xl);
 		padding: var(--padding-lg) !important;
-		background-color: var(--bg-color);
+		background-color: var(--bg-light-gray);
 		box-shadow: var(--card-shadow) ;
 
 		&.edit-mode:hover {
@@ -463,7 +463,7 @@ body {
 					height: 20px;
 					width: 20px;
 					color: var(--text-on-light-gray);
-					background-color: var(--fg-color);
+					background-color: var(--bg-light-gray);
 
 					margin-right: var(--margin-sm);
 					border-radius: var(--border-radius-full);

From ada2e20af3e6bffb95edfe8b80e57e80648c778f Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Mon, 31 Jul 2023 16:42:31 +0530
Subject: [PATCH 58/60] fix: has_permission check on int pk

---
 frappe/permissions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/permissions.py b/frappe/permissions.py
index ae26b56b2e..e71e2be20f 100644
--- a/frappe/permissions.py
+++ b/frappe/permissions.py
@@ -89,7 +89,7 @@ def has_permission(
 	meta = frappe.get_meta(doctype)
 
 	if doc:
-		if isinstance(doc, str):
+		if isinstance(doc, (str, int)):
 			doc = frappe.get_doc(meta.name, doc)
 		perm = get_doc_permissions(doc, user=user, ptype=ptype).get(ptype)
 		if not perm:

From 2c92043998019ac2b1c5dfa9d5548e822bf9efc5 Mon Sep 17 00:00:00 2001
From: Corentin Flr <10946971+cogk@users.noreply.github.com>
Date: Mon, 31 Jul 2023 18:10:53 +0200
Subject: [PATCH 59/60] fix(doctype): Allow cached_property decorator in
 controllers (#21881)

---
 frappe/core/doctype/doctype/doctype.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py
index 65f0b826b9..f7e6f28527 100644
--- a/frappe/core/doctype/doctype/doctype.py
+++ b/frappe/core/doctype/doctype/doctype.py
@@ -240,9 +240,7 @@ class DocType(Document):
 			controller = Document
 
 		available_objects = {x for x in dir(controller) if isinstance(x, str)}
-		property_set = {
-			x for x in available_objects if isinstance(getattr(controller, x, None), property)
-		}
+		property_set = {x for x in available_objects if is_a_property(getattr(controller, x, None))}
 		method_set = {
 			x for x in available_objects if x not in property_set and callable(getattr(controller, x, None))
 		}
@@ -1795,13 +1793,18 @@ def make_module_and_roles(doc, perm_fieldname="permissions"):
 			raise
 
 
+def is_a_property(x) -> bool:
+	"""Get properties (@property, @cached_property) in a controller class"""
+	from functools import cached_property
+
+	return isinstance(x, (property, cached_property))
+
+
 def check_fieldname_conflicts(docfield):
 	"""Checks if fieldname conflicts with methods or properties"""
 	doc = frappe.get_doc({"doctype": docfield.dt})
 	available_objects = [x for x in dir(doc) if isinstance(x, str)]
-	property_list = [
-		x for x in available_objects if isinstance(getattr(type(doc), x, None), property)
-	]
+	property_list = [x for x in available_objects if is_a_property(getattr(type(doc), x, None))]
 	method_list = [
 		x for x in available_objects if x not in property_list and callable(getattr(doc, x))
 	]

From 8bf8950c2b49025b6c7269509fc9b733481fdaef Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Tue, 1 Aug 2023 20:59:46 +0530
Subject: [PATCH 60/60] fix: include server script in recorder stack trace
 (#21891)

[skip ci]
---
 frappe/recorder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/recorder.py b/frappe/recorder.py
index 96ab502fec..ac1c4951ac 100644
--- a/frappe/recorder.py
+++ b/frappe/recorder.py
@@ -45,7 +45,7 @@ def get_current_stack_frames():
 		current = inspect.currentframe()
 		frames = inspect.getouterframes(current, context=10)
 		for frame, filename, lineno, function, context, index in list(reversed(frames))[:-2]:
-			if "/apps/" in filename:
+			if "/apps/" in filename or "<serverscript>" in filename:
 				yield {
 					"filename": TRACEBACK_PATH_PATTERN.sub("", filename),
 					"lineno": lineno,