Merge branch 'develop' into fix-field-replace

.github/helper/install.sh

@@ -54,8 +54,6 @@ fi
 
 echo "Starting Bench..."
 
-export FRAPPE_TUNE_GC=True
-
 bench start &> ~/frappe-bench/bench_start.log &
 
 if [ "$TYPE" == "server" ]

frappe/__init__.py

@@ -58,7 +58,7 @@ re._MAXCACHE = (
 	50  # reduced from default 512 given we are already maintaining this on parent worker
 )
 
-_tune_gc = bool(os.environ.get("FRAPPE_TUNE_GC", False))
+_tune_gc = bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True)))
 
 if _dev_server:
 	warnings.simplefilter("always", DeprecationWarning)

frappe/app.py

@@ -33,12 +33,17 @@ _sites_path = os.environ.get("SITES_PATH", ".")
 
 # If gc.freeze is done then importing modules before forking allows us to share the memory
 if frappe._tune_gc:
+	import bleach
+	import pydantic
+
 	import frappe.boot
 	import frappe.client
+	import frappe.core.doctype.file.file
 	import frappe.core.doctype.user.user
 	import frappe.database.mariadb.database  # Load database related utils
 	import frappe.database.query
 	import frappe.desk.desktop  # workspace
+	import frappe.desk.form.save
 	import frappe.model.db_query
 	import frappe.query_builder
 	import frappe.utils.background_jobs  # Enqueue is very common

frappe/auth.py

@@ -349,8 +349,6 @@ class CookieManager:
 		expires = datetime.datetime.now() + datetime.timedelta(days=3)
 		if frappe.session.sid:
 			self.set_cookie("sid", frappe.session.sid, expires=expires, httponly=True)
-		if frappe.session.session_country:
-			self.set_cookie("country", frappe.session.session_country)
 
 	def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"):
 		if not secure and hasattr(frappe.local, "request"):

frappe/core/doctype/access_log/access_log.py

@@ -59,7 +59,7 @@ def _make_access_log(
 	user = frappe.session.user
 	in_request = frappe.request and frappe.request.method == "GET"
 
-	frappe.get_doc(
+	access_log = frappe.get_doc(
 		{
 			"doctype": "Access Log",
 			"user": user,
@@ -72,7 +72,12 @@ def _make_access_log(
 			"filters": cstr(filters) or None,
 			"columns": columns,
 		}
-	).db_insert()
+	)
+
+	if frappe.flags.read_only:
+		access_log.deferred_insert()
+	else:
+		access_log.db_insert()
 
 	# `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview`
 	# dont commit in test mode. It must be tempting to put this block along with the in_request in the

frappe/core/doctype/file/file.py

@@ -724,7 +724,7 @@ def has_permission(doc, ptype=None, user=None):
 	if ptype == "create":
 		return frappe.has_permission("File", "create", user=user)
 
-	if not doc.is_private or doc.owner == user or user == "Administrator":
+	if not doc.is_private or (user != "Guest" and doc.owner == user) or user == "Administrator":
 		return True
 
 	if doc.attached_to_doctype and doc.attached_to_name:

frappe/core/doctype/file/test_file.py

@@ -29,11 +29,11 @@ test_content1 = "Hello"
 test_content2 = "Hello World"
 
 
-def make_test_doc():
+def make_test_doc(ignore_permissions=False):
 	d = frappe.new_doc("ToDo")
 	d.description = "Test"
 	d.assigned_by = frappe.session.user
-	d.save()
+	d.save(ignore_permissions)
 	return d.doctype, d.name
 
 
@@ -785,3 +785,81 @@ class TestFileOptimization(FrappeTestCase):
 			file_content = f.read()
 
 		self.assertEqual(get_extension("", None, file_content), "jpg")
+
+
+class TestGuestFileAndAttachments(FrappeTestCase):
+	def setUp(self) -> None:
+		frappe.db.delete("File", {"is_folder": 0})
+		frappe.get_doc(
+			doctype="DocType",
+			name="Test For Attachment",
+			module="Custom",
+			custom=1,
+			fields=[
+				{"label": "Title", "fieldname": "title", "fieldtype": "Data"},
+				{"label": "Attachment", "fieldname": "attachment", "fieldtype": "Attach"},
+			],
+		).insert(ignore_if_duplicate=True)
+
+	def tearDown(self) -> None:
+		frappe.set_user("Administrator")
+		frappe.db.rollback()
+		frappe.delete_doc("DocType", "Test For Attachment")
+
+	def test_attach_unattached_guest_file(self):
+		"""Ensure that unattached files are attached on doc update."""
+		f = frappe.new_doc(
+			"File",
+			file_name="test_private_guest_attachment.txt",
+			content="Guest Home",
+			is_private=1,
+		).insert(ignore_permissions=True)
+
+		d = frappe.new_doc("Test For Attachment")
+		d.title = "Test for attachment on update"
+		d.attachment = f.file_url
+		d.assigned_by = frappe.session.user
+		d.save()
+
+		self.assertTrue(
+			frappe.db.exists(
+				"File",
+				{
+					"file_name": "test_private_guest_attachment.txt",
+					"file_url": f.file_url,
+					"attached_to_doctype": "Test For Attachment",
+					"attached_to_name": d.name,
+					"attached_to_field": "attachment",
+				},
+			)
+		)
+
+	def test_list_private_guest_single_file(self):
+		"""Ensure that guests are not able to read private standalone guest files."""
+		frappe.set_user("Guest")
+
+		file = frappe.new_doc(
+			"File",
+			file_name="test_private_guest_single_txt",
+			content="Private single File",
+			is_private=1,
+		).insert(ignore_permissions=True)
+
+		self.assertFalse(file.is_downloadable())
+
+	def test_list_private_guest_attachment(self):
+		"""Ensure that guests are not able to read private guest attachments."""
+		frappe.set_user("Guest")
+
+		self.attached_to_doctype, self.attached_to_docname = make_test_doc(ignore_permissions=True)
+
+		file = frappe.new_doc(
+			"File",
+			file_name="test_private_guest_attachment.txt",
+			attached_to_doctype=self.attached_to_doctype,
+			attached_to_name=self.attached_to_docname,
+			content="Private Attachment",
+			is_private=1,
+		).insert(ignore_permissions=True)
+
+		self.assertFalse(file.is_downloadable())

frappe/core/doctype/file/utils.py

@@ -294,10 +294,11 @@ def update_existing_file_docs(doc: "File") -> None:
 	).run()
 
 
-def attach_files_to_document(doc: "File", event) -> None:
+def attach_files_to_document(doc: "Document", event) -> None:
 	"""Runs on on_update hook of all documents.
-	Goes through every Attach and Attach Image field and attaches
-	the file url to the document if it is not already attached.
+	Goes through every file linked with the Attach and Attach Image field and attaches
+	the file to the document if not already attached. If no file is found, a new file
+	is created.
 	"""
 
 	attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]})
@@ -320,6 +321,28 @@ def attach_files_to_document(doc: "File", event) -> None:
 		):
 			return
 
+		unattached_file = frappe.db.exists(
+			"File",
+			{
+				"file_url": value,
+				"attached_to_name": None,
+				"attached_to_doctype": None,
+				"attached_to_field": None,
+			},
+		)
+
+		if unattached_file:
+			frappe.db.set_value(
+				"File",
+				unattached_file,
+				field={
+					"attached_to_name": doc.name,
+					"attached_to_doctype": doc.doctype,
+					"attached_to_field": df.fieldname,
+				},
+			)
+			return
+
 		file: "File" = frappe.get_doc(
 			doctype="File",
 			file_url=value,

frappe/core/doctype/package_import/package_import.py

@@ -10,6 +10,7 @@ from frappe.desk.form.load import get_attachments
 from frappe.model.document import Document
 from frappe.model.sync import get_doc_files
 from frappe.modules.import_file import import_doc, import_file_by_path
+from frappe.utils import get_files_path
 
 
 class PackageImport(Document):
@@ -35,7 +36,7 @@ class PackageImport(Document):
 			[
 				"tar",
 				"xzf",
-				frappe.get_site_path(attachment.file_url.strip("/")),
+				get_files_path(attachment.file_name, is_private=attachment.is_private),
 				"-C",
 				frappe.get_site_path("packages"),
 			]

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -97,10 +97,13 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 
 	handle_enter_press(e) {
 		if (e.which === frappe.ui.keyCode.ENTER) {
-			var $target = $(e.target);
-			if ($target.hasClass("prev-btn")) {
+			let $target = $(e.target);
+			if ($target.hasClass("prev-btn") || $target.hasClass("next-btn")) {
 				$target.trigger("click");
 			} else {
+				// hitting enter on autocomplete field shouldn't trigger next slide.
+				if ($target.data().fieldtype == "Autocomplete") return;
+
 				this.container.find(".next-btn").trigger("click");
 				e.preventDefault();
 			}
@@ -545,15 +548,19 @@ frappe.setup.utils = {
 
 		slide.get_input("timezone").empty().add_options(data.all_timezones);
 
-		// set values if present
-		if (frappe.wizard.values.country) {
-			country_field.set_input(frappe.wizard.values.country);
-		} else if (data.default_country) {
-			country_field.set_input(data.default_country);
-		}
-
 		slide.get_field("currency").set_input(frappe.wizard.values.currency);
 		slide.get_field("timezone").set_input(frappe.wizard.values.timezone);
+
+		// set values if present
+		let country =
+			frappe.wizard.values.country ||
+			data.default_country ||
+			guess_country(frappe.setup.data.regional_data.country_info);
+
+		if (country) {
+			country_field.set_input(country);
+			$(country_field.input).change();
+		}
 	},
 
 	bind_language_events: function (slide) {
@@ -630,3 +637,16 @@ frappe.setup.utils = {
 		});
 	},
 };
+
+function guess_country(country_info) {
+	try {
+		const system_timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+		for ([country, info] of Object.entries(country_info)) {
+			let possible_timezones = (info.timezones || []).filter((t) => t == system_timezone);
+			if (possible_timezones.length) return country;
+		}
+	} catch (e) {
+		console.log("Could not guess country", e);
+	}
+}

frappe/desk/search.py

@@ -245,10 +245,6 @@ def search_widget(
 def get_std_fields_list(meta, key):
 	# get additional search fields
 	sflist = ["name"]
-	if meta.search_fields:
-		for d in meta.search_fields.split(","):
-			if d.strip() not in sflist:
-				sflist.append(d.strip())
 
 	if meta.title_field and meta.title_field not in sflist:
 		sflist.append(meta.title_field)
@@ -256,6 +252,11 @@ def get_std_fields_list(meta, key):
 	if key not in sflist:
 		sflist.append(key)
 
+	if meta.search_fields:
+		for d in meta.search_fields.split(","):
+			if d.strip() not in sflist:
+				sflist.append(d.strip())
+
 	return sflist
 
 

frappe/email/queue.py

@@ -149,7 +149,7 @@ def flush(from_test=False):
 				now=from_test,
 				job_id=f"email_queue_sendmail_{row.name}",
 				queue="short",
-				dedupicate=True,
+				deduplicate=True,
 			)
 		except Exception:
 			frappe.get_doc("Email Queue", row.name).log_error()

frappe/public/js/frappe/form/formatters.js

@@ -352,7 +352,9 @@ frappe.form.formatters = {
 		const link_field = meta.fields.find((df) => df.fieldtype === "Link");
 		const formatted_values = rows.map((row) => {
 			const value = row[link_field.fieldname];
-			return frappe.format(value, link_field, options, row);
+			return `<span class="text-nowrap">
+				${frappe.format(value, link_field, options, row)}
+			</span>`;
 		});
 		return formatted_values.join(", ");
 	},

frappe/sessions.py

@@ -236,9 +236,6 @@ class Session:
 					"session_expiry": get_expiry_period(),
 					"full_name": self.full_name,
 					"user_type": self.user_type,
-					"session_country": get_geo_ip_country(frappe.local.request_ip)
-					if frappe.local.request_ip
-					else None,
 				}
 			)
 

frappe/tests/test_perf.py

@@ -74,6 +74,9 @@ class TestPerformance(FrappeTestCase):
 	def test_get_value_limits(self):
 		# check both dict and list style filters
 		filters = [{"enabled": 1}, [["enabled", "=", 1]]]
+
+		# Warm up code
+		frappe.db.get_values("User", filters=filters[0], limit=1)
 		for filter in filters:
 			with self.assertRowsRead(1):
 				self.assertEqual(1, len(frappe.db.get_values("User", filters=filter, limit=1)))

frappe/tests/test_utils.py

@@ -1163,3 +1163,44 @@ class TestRounding(FrappeTestCase):
 
 	def test_default_rounding(self):
 		self.assertEqual(frappe.get_system_settings("rounding_method"), "Banker's Rounding")
+
+
+class TestTypingValidations(FrappeTestCase):
+	def test_validate_argument_types(self):
+		from frappe.core.doctype.doctype.doctype import DocType
+		from frappe.utils.typing_validations import FrappeTypeError, validate_argument_types
+
+		@validate_argument_types
+		def test_simple_types(a: int, b: float, c: bool):
+			return a, b, c
+
+		@validate_argument_types
+		def test_sequence(a: str, b: list[dict] | None = None, c: dict[str, int] | None = None):
+			return a, b, c
+
+		@validate_argument_types
+		def test_doctypes(a: DocType | dict):
+			return a
+
+		self.assertEqual(test_simple_types(True, 2.0, True), (1, 2.0, True))
+		self.assertEqual(test_simple_types(1, 2, 1), (1, 2.0, True))
+		self.assertEqual(test_simple_types(1.0, 2, 1), (1, 2.0, True))
+		self.assertEqual(test_simple_types(1, 2, "1"), (1, 2.0, True))
+		with self.assertRaises(FrappeTypeError):
+			test_simple_types(1, 2, "a")
+		with self.assertRaises(FrappeTypeError):
+			test_simple_types(1, 2, None)
+
+		self.assertEqual(test_sequence("a", [{"a": 1}], {"a": 1}), ("a", [{"a": 1}], {"a": 1}))
+		self.assertEqual(test_sequence("a", None, None), ("a", None, None))
+		self.assertEqual(test_sequence("a", [{"a": 1}], None), ("a", [{"a": 1}], None))
+		self.assertEqual(test_sequence("a", None, {"a": 1}), ("a", None, {"a": 1}))
+		self.assertEqual(test_sequence("a", [{"a": 1}], {"a": "1.0"}), ("a", [{"a": 1}], {"a": 1}))
+		with self.assertRaises(FrappeTypeError):
+			test_sequence("a", [{"a": 1}], True)
+
+		doctype = frappe.get_last_doc("DocType")
+		self.assertEqual(test_doctypes(doctype), doctype)
+		self.assertEqual(test_doctypes(doctype.as_dict()), doctype.as_dict())
+		with self.assertRaises(FrappeTypeError):
+			test_doctypes("a")

frappe/utils/connections.py

@@ -31,10 +31,9 @@ def check_redis(redis_services=None):
 	config = get_conf()
 	services = redis_services or REDIS_KEYS
 	status = {}
-	for conn in services:
-		redis_url = urlparse(config.get(conn)).netloc
-		redis_host, redis_port = redis_url.split(":")
-		status[conn] = is_open(redis_host, redis_port)
+	for srv in services:
+		url = urlparse(config[srv])
+		status[srv] = is_open(url.hostname, url.port)
 	return status
 
 

frappe/utils/typing_validations.py

@@ -1,11 +1,7 @@
 from functools import lru_cache, wraps
 from inspect import _empty, isclass, signature
 from types import EllipsisType
-from typing import Any, Callable, ForwardRef, TypeVar, Union
-
-from pydantic.config import BaseConfig
-from pydantic.error_wrappers import ValidationError as PyValidationError
-from pydantic.tools import NameFactory, _generate_parsing_type_name
+from typing import Callable, ForwardRef, TypeVar, Union
 
 from frappe.exceptions import FrappeTypeError
 
@@ -69,29 +65,10 @@ def raise_type_error(
 
 
 @lru_cache(maxsize=2048)
-def _get_parsing_type(
-	type_: Any, *, type_name: NameFactory | None = None, config: type[BaseConfig] = None
-) -> Any:
-	# Note: this is a copy of pydantic.tools._get_parsing_type with the addition of allowing a config argument
-	from pydantic.main import create_model
+def TypeAdapter(type_):
+	from pydantic import TypeAdapter as PyTypeAdapter
 
-	if type_name is None:
-		type_name = _generate_parsing_type_name
-	if not isinstance(type_name, str):
-		type_name = type_name(type_)
-	return create_model(type_name, __root__=(type_, ...), __config__=config)
-
-
-def parse_obj_as(
-	type_: type[T],
-	obj: Any,
-	*,
-	type_name: NameFactory | None = None,
-	config: type[BaseConfig] | None = None,
-) -> T:
-	# Note: This is a copy of pydantic.tools.parse_obj_as with the addition of allowing a config argument
-	model_type = _get_parsing_type(type_, type_name=type_name, config=config)  # type: ignore[arg-type]
-	return model_type(__root__=obj).__root__
+	return PyTypeAdapter(type_, config=FrappePydanticConfig)
 
 
 def transform_parameter_types(func: Callable, args: tuple, kwargs: dict):
@@ -103,6 +80,8 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict):
 	if not (args or kwargs) or not func.__annotations__:
 		return args, kwargs
 
+	from pydantic import ValidationError as PyValidationError
+
 	annotations = func.__annotations__
 	new_args, new_kwargs = list(args), kwargs
 
@@ -157,9 +136,7 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict):
 
 		# validate the type set using pydantic - raise a TypeError if Validation is raised or Ellipsis is returned
 		try:
-			current_arg_value_after = parse_obj_as(
-				current_arg_type, current_arg_value, type_name=current_arg, config=FrappePydanticConfig
-			)
+			current_arg_value_after = TypeAdapter(current_arg_type).validate_python(current_arg_value)
 		except (TypeError, PyValidationError) as e:
 			raise_type_error(current_arg, current_arg_type, current_arg_value, current_exception=e)
 

pyproject.toml

@@ -53,7 +53,7 @@ dependencies = [
     "psycopg2-binary~=2.9.1",
     "pyOpenSSL~=23.2.0",
     "pycryptodome~=3.18.0",
-    "pydantic~=1.10.8",
+    "pydantic==2.0",
     "pyotp~=2.8.0",
     "python-dateutil~=2.8.2",
     "pytz==2023.3",