From 1a7b7a589b90899377c46c18819b5bde7c12658b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 16:53:25 +0530 Subject: [PATCH 01/28] build(deps): Bump Pydantic from v1 to v2 --- frappe/utils/typing_validations.py | 36 +++++------------------------- pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/frappe/utils/typing_validations.py b/frappe/utils/typing_validations.py index e7ebcfbdff..4c43064af6 100644 --- a/frappe/utils/typing_validations.py +++ b/frappe/utils/typing_validations.py @@ -1,11 +1,10 @@ from functools import lru_cache, wraps from inspect import _empty, isclass, signature from types import EllipsisType -from typing import Any, Callable, ForwardRef, TypeVar, Union +from typing import Callable, ForwardRef, TypeVar, Union -from pydantic.config import BaseConfig -from pydantic.error_wrappers import ValidationError as PyValidationError -from pydantic.tools import NameFactory, _generate_parsing_type_name +from pydantic import TypeAdapter as PyTypeAdapter +from pydantic import ValidationError as PyValidationError from frappe.exceptions import FrappeTypeError @@ -69,29 +68,8 @@ def raise_type_error( @lru_cache(maxsize=2048) -def _get_parsing_type( - type_: Any, *, type_name: NameFactory | None = None, config: type[BaseConfig] = None -) -> Any: - # Note: this is a copy of pydantic.tools._get_parsing_type with the addition of allowing a config argument - from pydantic.main import create_model - - if type_name is None: - type_name = _generate_parsing_type_name - if not isinstance(type_name, str): - type_name = type_name(type_) - return create_model(type_name, __root__=(type_, ...), __config__=config) - - -def parse_obj_as( - type_: type[T], - obj: Any, - *, - type_name: NameFactory | None = None, - config: type[BaseConfig] | None = None, -) -> T: - # Note: This is a copy of pydantic.tools.parse_obj_as with the addition of allowing a config argument - model_type = _get_parsing_type(type_, type_name=type_name, config=config) # type: ignore[arg-type] - return model_type(__root__=obj).__root__ +def TypeAdapter(type_): + return PyTypeAdapter(type_) def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): @@ -157,9 +135,7 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): # validate the type set using pydantic - raise a TypeError if Validation is raised or Ellipsis is returned try: - current_arg_value_after = parse_obj_as( - current_arg_type, current_arg_value, type_name=current_arg, config=FrappePydanticConfig - ) + current_arg_value_after = TypeAdapter(current_arg_type).validate_python(current_arg_value) except (TypeError, PyValidationError) as e: raise_type_error(current_arg, current_arg_type, current_arg_value, current_exception=e) diff --git a/pyproject.toml b/pyproject.toml index aa89eed928..dde346e7aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "psycopg2-binary~=2.9.1", "pyOpenSSL~=23.0.0", "pycryptodome~=3.10.1", - "pydantic~=1.10.2", + "pydantic~=2.0b2", "pyotp~=2.6.0", "python-dateutil~=2.8.1", "pytz==2022.1", From 5c2fe7c292262a030c1ad6894f8fec65b7388f1b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 17:17:13 +0530 Subject: [PATCH 02/28] test: Add tests for validate_argument_types decorator --- frappe/tests/test_utils.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 4b362e7b47..e46e37b518 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1155,3 +1155,35 @@ class TestRounding(FrappeTestCase): def test_default_rounding(self): self.assertEqual(frappe.get_system_settings("rounding_method"), "Banker's Rounding") + + +class TestTypingValidations(FrappeTestCase): + def test_validate_argument_types(self): + from frappe.utils.typing_validations import FrappeTypeError, validate_argument_types + + @validate_argument_types + def test_simple_types(a: int, b: float, c: bool): + return a, b, c + + @validate_argument_types + def test_sequence(a: str, b: list[dict] | None = None, c: dict[str, int] | None = None): + return a, b, c + + self.assertEqual(test_simple_types(True, 2.0, True), (1, 2.0, True)) + self.assertEqual(test_simple_types(1, 2, 1), (1, 2.0, True)) + self.assertEqual(test_simple_types(1.0, 2, 1), (1, 2.0, True)) + self.assertEqual(test_simple_types(1, 2, "1"), (1, 2.0, True)) + + with self.assertRaises(FrappeTypeError): + test_simple_types(1, 2, "a") + with self.assertRaises(FrappeTypeError): + test_simple_types(1, 2, None) + + self.assertEqual(test_sequence("a", [{"a": 1}], {"a": 1}), ("a", [{"a": 1}], {"a": 1})) + self.assertEqual(test_sequence("a", None, None), ("a", None, None)) + self.assertEqual(test_sequence("a", [{"a": 1}], None), ("a", [{"a": 1}], None)) + self.assertEqual(test_sequence("a", None, {"a": 1}), ("a", None, {"a": 1})) + self.assertEqual(test_sequence("a", [{"a": 1}], {"a": "1.0"}), ("a", [{"a": 1}], {"a": 1})) + + with self.assertRaises(FrappeTypeError): + test_sequence("a", [{"a": 1}], True) From ae3a61b3fc75957d2e4c710179338d5784991049 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 19:26:32 +0530 Subject: [PATCH 03/28] test: Add tests for Document arg validations --- frappe/tests/test_utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index e46e37b518..2e83f8b8b6 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1159,6 +1159,7 @@ class TestRounding(FrappeTestCase): class TestTypingValidations(FrappeTestCase): def test_validate_argument_types(self): + from frappe.core.doctype.doctype.doctype import DocType from frappe.utils.typing_validations import FrappeTypeError, validate_argument_types @validate_argument_types @@ -1169,11 +1170,14 @@ class TestTypingValidations(FrappeTestCase): def test_sequence(a: str, b: list[dict] | None = None, c: dict[str, int] | None = None): return a, b, c + @validate_argument_types + def test_doctypes(a: DocType | dict): + return a + self.assertEqual(test_simple_types(True, 2.0, True), (1, 2.0, True)) self.assertEqual(test_simple_types(1, 2, 1), (1, 2.0, True)) self.assertEqual(test_simple_types(1.0, 2, 1), (1, 2.0, True)) self.assertEqual(test_simple_types(1, 2, "1"), (1, 2.0, True)) - with self.assertRaises(FrappeTypeError): test_simple_types(1, 2, "a") with self.assertRaises(FrappeTypeError): @@ -1184,6 +1188,11 @@ class TestTypingValidations(FrappeTestCase): self.assertEqual(test_sequence("a", [{"a": 1}], None), ("a", [{"a": 1}], None)) self.assertEqual(test_sequence("a", None, {"a": 1}), ("a", None, {"a": 1})) self.assertEqual(test_sequence("a", [{"a": 1}], {"a": "1.0"}), ("a", [{"a": 1}], {"a": 1})) - with self.assertRaises(FrappeTypeError): test_sequence("a", [{"a": 1}], True) + + doctype = frappe.get_last_doc("DocType") + self.assertEqual(test_doctypes(doctype), doctype) + self.assertEqual(test_doctypes(doctype.as_dict()), doctype.as_dict()) + with self.assertRaises(FrappeTypeError): + test_doctypes("a") From ee3d4da734dc0fd646076181fc43e0e4d57a8ce0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 19:26:57 +0530 Subject: [PATCH 04/28] fix: Allow arbitrary objects validation in TypeAdapter --- frappe/utils/typing_validations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/typing_validations.py b/frappe/utils/typing_validations.py index 4c43064af6..83f59fb2ec 100644 --- a/frappe/utils/typing_validations.py +++ b/frappe/utils/typing_validations.py @@ -69,7 +69,7 @@ def raise_type_error( @lru_cache(maxsize=2048) def TypeAdapter(type_): - return PyTypeAdapter(type_) + return PyTypeAdapter(type_, config=FrappePydanticConfig) def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): From 553e8622b5876f4ed5e38869343d0666a535850a Mon Sep 17 00:00:00 2001 From: David Arnold Date: Sun, 25 Jun 2023 19:46:08 -0500 Subject: [PATCH 05/28] fix: don't break link list in middle of link in table multiselect --- frappe/public/js/frappe/form/formatters.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index beddbf512d..81e403169f 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -352,7 +352,11 @@ frappe.form.formatters = { const link_field = meta.fields.find((df) => df.fieldtype === "Link"); const formatted_values = rows.map((row) => { const value = row[link_field.fieldname]; - return frappe.format(value, link_field, options, row); + return ( + `` + + frappe.format(value, link_field, options, row) + + `` + ); }); return formatted_values.join(", "); }, From 58fc6a2b940200d75ad3278cd9d5baaa6e8a97d7 Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 30 Jun 2023 22:18:45 +0530 Subject: [PATCH 06/28] build: Bump to stable Pydantic v2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4d61d6845..17a75f3c0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "psycopg2-binary~=2.9.1", "pyOpenSSL~=23.2.0", "pycryptodome~=3.18.0", - "pydantic~=2.0b3", + "pydantic==2.0", "pyotp~=2.8.0", "python-dateutil~=2.8.2", "pytz==2023.3", From e72191a56ffdac96ba4bbf048926a59df57a4de8 Mon Sep 17 00:00:00 2001 From: anandbaburajan Date: Sat, 1 Jul 2023 00:45:35 +0530 Subject: [PATCH 07/28] fix: attach unattached guest files on doc update --- frappe/core/doctype/file/utils.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index 1d0d145303..5d8c4503bb 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -320,6 +320,28 @@ def attach_files_to_document(doc: "File", event) -> None: ): return + unattached_file = frappe.db.exists( + "File", + { + "file_url": value, + "attached_to_name": None, + "attached_to_doctype": None, + "attached_to_field": None, + }, + ) + + if unattached_file: + frappe.db.set_value( + "File", + unattached_file, + field={ + "attached_to_name": doc.name, + "attached_to_doctype": doc.doctype, + "attached_to_field": df.fieldname, + }, + ) + return + file: "File" = frappe.get_doc( doctype="File", file_url=value, From d4c30c29f693f4df7ce7bdf1768e261129b3d2d3 Mon Sep 17 00:00:00 2001 From: anandbaburajan Date: Sat, 1 Jul 2023 00:50:59 +0530 Subject: [PATCH 08/28] fix: don't show guest's private files to other guests --- frappe/core/doctype/file/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 8778826b56..359425d58c 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -724,7 +724,7 @@ def has_permission(doc, ptype=None, user=None): if ptype == "create": return frappe.has_permission("File", "create", user=user) - if not doc.is_private or doc.owner == user or user == "Administrator": + if not doc.is_private or (user != "Guest" and doc.owner == user) or user == "Administrator": return True if doc.attached_to_doctype and doc.attached_to_name: From d36765457d56705bdf7bacf070f8578092920987 Mon Sep 17 00:00:00 2001 From: anandbaburajan Date: Sat, 1 Jul 2023 19:41:22 +0530 Subject: [PATCH 09/28] test: add tests for guest files --- frappe/core/doctype/file/test_file.py | 78 +++++++++++++++++++++++++++ frappe/core/doctype/file/utils.py | 2 +- 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 1e7e698062..e442a12b6d 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -785,3 +785,81 @@ class TestFileOptimization(FrappeTestCase): file_content = f.read() self.assertEqual(get_extension("", None, file_content), "jpg") + + +class TestGuestFileAndAttachments(FrappeTestCase): + def setUp(self) -> None: + frappe.db.delete("File", {"is_folder": 0}) + frappe.get_doc( + doctype="DocType", + name="Test For Attachment", + module="Custom", + custom=1, + fields=[ + {"label": "Title", "fieldname": "title", "fieldtype": "Data"}, + {"label": "Attachment", "fieldname": "attachment", "fieldtype": "Attach"}, + ], + ).insert(ignore_if_duplicate=True) + + def tearDown(self) -> None: + frappe.set_user("Administrator") + frappe.db.rollback() + frappe.delete_doc("DocType", "Test For Attachment") + + def test_attach_unattached_guest_file(self): + frappe.set_user("Guest") + + f = frappe.new_doc( + "File", + file_name="test_private_guest_attachment.txt", + content="Guest Home", + is_private=1, + ).insert() + + d = frappe.new_doc("Test For Attachment") + d.title = "Test for attachment on update" + d.attachment = f.file_url + d.assigned_by = frappe.session.user + d.save() + + self.assertTrue( + frappe.db.exists( + "File", + { + "file_name": "test_private_guest_attachment.txt", + "file_url": f.file_url, + "attached_to_doctype": "Test For Attachment", + "attached_to_name": d.name, + "attached_to_field": "attachment", + }, + ) + ) + + def test_list_private_guest_single_file(self): + """Ensure that guests are not able to list private standalone guest files.""" + frappe.set_user("Guest") + frappe.new_doc( + "File", + file_name="test_private_guest_single_txt", + content="Private single File", + is_private=1, + ).insert() + + files = [file.file_name for file in get_files_in_folder("Home")["files"]] + self.assertNotIn("test_private_guest_single_txt.txt", files) + + def test_list_private_guest_attachment(self): + """Ensure that guests are not able to list private guest attachments.""" + frappe.set_user("Guest") + self.attached_to_doctype, self.attached_to_docname = make_test_doc() + frappe.new_doc( + "File", + file_name="test_private_guest_attachment.txt", + attached_to_doctype=self.attached_to_doctype, + attached_to_name=self.attached_to_docname, + content="Private Attachment", + is_private=1, + ).insert() + + files = [file.file_name for file in get_files_in_folder("Home/Attachments")["files"]] + self.assertNotIn("test_private_guest_attachment.txt", files) diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index 5d8c4503bb..f1637c1906 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -294,7 +294,7 @@ def update_existing_file_docs(doc: "File") -> None: ).run() -def attach_files_to_document(doc: "File", event) -> None: +def attach_files_to_document(doc: "Document", event) -> None: """Runs on on_update hook of all documents. Goes through every Attach and Attach Image field and attaches the file url to the document if it is not already attached. From ad79c9d18062b544e06a607a68c31d7b8c7f4774 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 1 Jul 2023 20:06:00 +0530 Subject: [PATCH 10/28] chore: remove broken call to geoip This has never worked afaik --- frappe/auth.py | 2 -- frappe/sessions.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 29c3e41694..d1259e1aaf 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -349,8 +349,6 @@ class CookieManager: expires = datetime.datetime.now() + datetime.timedelta(days=3) if frappe.session.sid: self.set_cookie("sid", frappe.session.sid, expires=expires, httponly=True) - if frappe.session.session_country: - self.set_cookie("country", frappe.session.session_country) def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"): if not secure and hasattr(frappe.local, "request"): diff --git a/frappe/sessions.py b/frappe/sessions.py index 2709de8fab..b0eb7e7353 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -236,9 +236,6 @@ class Session: "session_expiry": get_expiry_period(), "full_name": self.full_name, "user_type": self.user_type, - "session_country": get_geo_ip_country(frappe.local.request_ip) - if frappe.local.request_ip - else None, } ) From 2481b7035e8fe5c2d2885aeb82d53f65ed323694 Mon Sep 17 00:00:00 2001 From: anandbaburajan Date: Sat, 1 Jul 2023 22:32:34 +0530 Subject: [PATCH 11/28] chore: fix permission issues in tests and update docstring of attach_files_to_document --- frappe/core/doctype/file/test_file.py | 32 +++++++++++++-------------- frappe/core/doctype/file/utils.py | 5 +++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index e442a12b6d..ef3a7a5f2b 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -29,11 +29,11 @@ test_content1 = "Hello" test_content2 = "Hello World" -def make_test_doc(): +def make_test_doc(ignore_permissions=False): d = frappe.new_doc("ToDo") d.description = "Test" d.assigned_by = frappe.session.user - d.save() + d.save(ignore_permissions) return d.doctype, d.name @@ -807,14 +807,13 @@ class TestGuestFileAndAttachments(FrappeTestCase): frappe.delete_doc("DocType", "Test For Attachment") def test_attach_unattached_guest_file(self): - frappe.set_user("Guest") - + """Ensure that unattached files are attached on doc update.""" f = frappe.new_doc( "File", file_name="test_private_guest_attachment.txt", content="Guest Home", is_private=1, - ).insert() + ).insert(ignore_permissions=True) d = frappe.new_doc("Test For Attachment") d.title = "Test for attachment on update" @@ -836,30 +835,31 @@ class TestGuestFileAndAttachments(FrappeTestCase): ) def test_list_private_guest_single_file(self): - """Ensure that guests are not able to list private standalone guest files.""" + """Ensure that guests are not able to read private standalone guest files.""" frappe.set_user("Guest") - frappe.new_doc( + + file = frappe.new_doc( "File", file_name="test_private_guest_single_txt", content="Private single File", is_private=1, - ).insert() + ).insert(ignore_permissions=True) - files = [file.file_name for file in get_files_in_folder("Home")["files"]] - self.assertNotIn("test_private_guest_single_txt.txt", files) + self.assertFalse(file.is_downloadable()) def test_list_private_guest_attachment(self): - """Ensure that guests are not able to list private guest attachments.""" + """Ensure that guests are not able to read private guest attachments.""" frappe.set_user("Guest") - self.attached_to_doctype, self.attached_to_docname = make_test_doc() - frappe.new_doc( + + self.attached_to_doctype, self.attached_to_docname = make_test_doc(ignore_permissions=True) + + file = frappe.new_doc( "File", file_name="test_private_guest_attachment.txt", attached_to_doctype=self.attached_to_doctype, attached_to_name=self.attached_to_docname, content="Private Attachment", is_private=1, - ).insert() + ).insert(ignore_permissions=True) - files = [file.file_name for file in get_files_in_folder("Home/Attachments")["files"]] - self.assertNotIn("test_private_guest_attachment.txt", files) + self.assertFalse(file.is_downloadable()) diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index f1637c1906..932fce9a03 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -296,8 +296,9 @@ def update_existing_file_docs(doc: "File") -> None: def attach_files_to_document(doc: "Document", event) -> None: """Runs on on_update hook of all documents. - Goes through every Attach and Attach Image field and attaches - the file url to the document if it is not already attached. + Goes through every file linked with the Attach and Attach Image field and attaches + the file to the document if not already attached. If no file is found, a new file + is created. """ attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}) From 70da4ba51ddc08bb1f03fc56ceff79f6b10d4a4a Mon Sep 17 00:00:00 2001 From: Kevin Shenk Date: Sat, 1 Jul 2023 13:56:50 -0400 Subject: [PATCH 12/28] fix: typo on enqueue args The deduplicate argument was missing the 'l'. --- frappe/email/queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 500a126f72..cae5f76b3d 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -149,7 +149,7 @@ def flush(from_test=False): now=from_test, job_id=f"email_queue_sendmail_{row.name}", queue="short", - dedupicate=True, + deduplicate=True, ) except Exception: frappe.get_doc("Email Queue", row.name).log_error() From e0f35fb85f1550086a4f37306d3700261a8f9121 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Sun, 2 Jul 2023 11:25:43 +0530 Subject: [PATCH 13/28] fix: Update country names (#21545) * fix: Update country names * fix: revert name back --- frappe/geo/country_info.json | 6 +++--- frappe/patches.txt | 1 + frappe/patches/v14_0/update_country_names.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 frappe/patches/v14_0/update_country_names.py diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json index f308dc63e3..858b43b883 100644 --- a/frappe/geo/country_info.json +++ b/frappe/geo/country_info.json @@ -669,7 +669,7 @@ "currency_fraction_units": 100, "isd": "+242" }, - "Congo, The Democratic Republic of the": { + "Congo, Democratic Republic of the": { "code": "cd", "number_format": "#,###.##", "currency": "CDF", @@ -2632,7 +2632,7 @@ ], "isd": "+992" }, - "Tanzania": { + "Tanzania, United Republic of": { "code": "tz", "currency": "TZS", "currency_name": "Tanzanian Shilling", @@ -2913,7 +2913,7 @@ "currency_fraction_units": 100, "isd": "+58" }, - "Vietnam": { + "Viet Nam": { "code": "vn", "currency": "VND", "currency_name": "Dong", diff --git a/frappe/patches.txt b/frappe/patches.txt index ebdda9b220..9ce2d0896d 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -226,3 +226,4 @@ frappe.desk.doctype.form_tour.patches.introduce_ui_tours execute:frappe.delete_doc_if_exists("Workspace", "Customization") execute:frappe.db.set_single_value("Document Naming Settings", "default_amend_naming", "Amend Counter") execute:frappe.delete_doc_if_exists("DocType", "Error Snapshot") +frappe.patches.v14_0.update_country_names diff --git a/frappe/patches/v14_0/update_country_names.py b/frappe/patches/v14_0/update_country_names.py new file mode 100644 index 0000000000..272c4ed896 --- /dev/null +++ b/frappe/patches/v14_0/update_country_names.py @@ -0,0 +1,12 @@ +import frappe + + +def execute(): + country_info_map = { + "Tanzania": "Tanzania, United Republic of", + "Vietnam": "Viet Nam", + "Congo, The Democratic Republic of the": "Congo, Democratic Republic of the", + } + + for old, new in country_info_map.items(): + frappe.rename_doc("Country", old, new, force=True) From 3a74b38e8c94d4b26267c650a9c9c112941f85e8 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Sun, 2 Jul 2023 12:08:43 +0530 Subject: [PATCH 14/28] Revert "fix: Update country names (#21545)" This reverts commit e0f35fb85f1550086a4f37306d3700261a8f9121. --- frappe/geo/country_info.json | 6 +++--- frappe/patches.txt | 1 - frappe/patches/v14_0/update_country_names.py | 12 ------------ 3 files changed, 3 insertions(+), 16 deletions(-) delete mode 100644 frappe/patches/v14_0/update_country_names.py diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json index 858b43b883..f308dc63e3 100644 --- a/frappe/geo/country_info.json +++ b/frappe/geo/country_info.json @@ -669,7 +669,7 @@ "currency_fraction_units": 100, "isd": "+242" }, - "Congo, Democratic Republic of the": { + "Congo, The Democratic Republic of the": { "code": "cd", "number_format": "#,###.##", "currency": "CDF", @@ -2632,7 +2632,7 @@ ], "isd": "+992" }, - "Tanzania, United Republic of": { + "Tanzania": { "code": "tz", "currency": "TZS", "currency_name": "Tanzanian Shilling", @@ -2913,7 +2913,7 @@ "currency_fraction_units": 100, "isd": "+58" }, - "Viet Nam": { + "Vietnam": { "code": "vn", "currency": "VND", "currency_name": "Dong", diff --git a/frappe/patches.txt b/frappe/patches.txt index 9ce2d0896d..ebdda9b220 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -226,4 +226,3 @@ frappe.desk.doctype.form_tour.patches.introduce_ui_tours execute:frappe.delete_doc_if_exists("Workspace", "Customization") execute:frappe.db.set_single_value("Document Naming Settings", "default_amend_naming", "Amend Counter") execute:frappe.delete_doc_if_exists("DocType", "Error Snapshot") -frappe.patches.v14_0.update_country_names diff --git a/frappe/patches/v14_0/update_country_names.py b/frappe/patches/v14_0/update_country_names.py deleted file mode 100644 index 272c4ed896..0000000000 --- a/frappe/patches/v14_0/update_country_names.py +++ /dev/null @@ -1,12 +0,0 @@ -import frappe - - -def execute(): - country_info_map = { - "Tanzania": "Tanzania, United Republic of", - "Vietnam": "Viet Nam", - "Congo, The Democratic Republic of the": "Congo, Democratic Republic of the", - } - - for old, new in country_info_map.items(): - frappe.rename_doc("Country", old, new, force=True) From ee4c66ecf152d82b33308ed5fb49370940a0eb34 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 1 Jul 2023 20:47:22 +0530 Subject: [PATCH 15/28] feat(UX): Guess country from system_timezone This is accurate enough for vast majority of countries. Alternate is using GeoIP and making network calls. --- frappe/desk/page/setup_wizard/setup_wizard.js | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js index 7d68fd683c..c19a336c3b 100644 --- a/frappe/desk/page/setup_wizard/setup_wizard.js +++ b/frappe/desk/page/setup_wizard/setup_wizard.js @@ -545,15 +545,19 @@ frappe.setup.utils = { slide.get_input("timezone").empty().add_options(data.all_timezones); - // set values if present - if (frappe.wizard.values.country) { - country_field.set_input(frappe.wizard.values.country); - } else if (data.default_country) { - country_field.set_input(data.default_country); - } - slide.get_field("currency").set_input(frappe.wizard.values.currency); slide.get_field("timezone").set_input(frappe.wizard.values.timezone); + + // set values if present + let country = + frappe.wizard.values.country || + data.default_country || + guess_country(frappe.setup.data.regional_data.country_info); + + if (country) { + country_field.set_input(country); + $(country_field.input).change(); + } }, bind_language_events: function (slide) { @@ -630,3 +634,16 @@ frappe.setup.utils = { }); }, }; + +function guess_country(country_info) { + try { + const system_timezone = Intl.DateTimeFormat().resolvedOptions().timeZone; + + for ([country, info] of Object.entries(country_info)) { + let possible_timezones = (info.timezones || []).filter((t) => t == system_timezone); + if (possible_timezones.length) return country; + } + } catch (e) { + console.log("Could not guess country", e); + } +} From 1a7cb47826e11d840e63b4c8aea1c2203795f61e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jul 2023 12:50:43 +0530 Subject: [PATCH 16/28] fix: setup wizard auto completes when clickin on autocomplete fields closes https://github.com/frappe/frappe/issues/15693 --- frappe/desk/page/setup_wizard/setup_wizard.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js index c19a336c3b..e5c268b24c 100644 --- a/frappe/desk/page/setup_wizard/setup_wizard.js +++ b/frappe/desk/page/setup_wizard/setup_wizard.js @@ -97,10 +97,13 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides { handle_enter_press(e) { if (e.which === frappe.ui.keyCode.ENTER) { - var $target = $(e.target); - if ($target.hasClass("prev-btn")) { + let $target = $(e.target); + if ($target.hasClass("prev-btn") || $target.hasClass("next-btn")) { $target.trigger("click"); } else { + // hitting enter on autocomplete field shouldn't trigger next slide. + if ($target.data().fieldtype == "Autocomplete") return; + this.container.find(".next-btn").trigger("click"); e.preventDefault(); } From 883445aefa77dd475a0d2ac27295723f17292621 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 2 Jul 2023 16:00:50 +0530 Subject: [PATCH 17/28] perf: Defer pydantic imports until function call Pydantic adds an additional 1-2MB in memory usage. We can defer it in case an environment doesn't use it at all. --- frappe/utils/typing_validations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/utils/typing_validations.py b/frappe/utils/typing_validations.py index 83f59fb2ec..91a318eae4 100644 --- a/frappe/utils/typing_validations.py +++ b/frappe/utils/typing_validations.py @@ -3,9 +3,6 @@ from inspect import _empty, isclass, signature from types import EllipsisType from typing import Callable, ForwardRef, TypeVar, Union -from pydantic import TypeAdapter as PyTypeAdapter -from pydantic import ValidationError as PyValidationError - from frappe.exceptions import FrappeTypeError SLACK_DICT = { @@ -69,6 +66,8 @@ def raise_type_error( @lru_cache(maxsize=2048) def TypeAdapter(type_): + from pydantic import TypeAdapter as PyTypeAdapter + return PyTypeAdapter(type_, config=FrappePydanticConfig) @@ -81,6 +80,8 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): if not (args or kwargs) or not func.__annotations__: return args, kwargs + from pydantic import ValidationError as PyValidationError + annotations = func.__annotations__ new_args, new_kwargs = list(args), kwargs From af7032893087db31f5eff6c31c9473561fdd4d0b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 2 Jul 2023 16:19:25 +0530 Subject: [PATCH 18/28] test: Bump memory usage treshold post pydantic v2 upgrade --- frappe/core/doctype/rq_job/test_rq_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index f5d5f89ed4..cb7faca187 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -160,7 +160,7 @@ class TestRQJob(FrappeTestCase): # If this starts failing analyze memory usage using memray or some equivalent tool to find # offending imports/function calls. # Refer this PR: https://github.com/frappe/frappe/pull/21467 - LAST_MEASURED_USAGE = 40 + LAST_MEASURED_USAGE = 42 self.assertLessEqual(rss, LAST_MEASURED_USAGE * 1.05, msg) From a63398778eb88c43a26f5985f40adbbcdd59e712 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jul 2023 16:22:14 +0530 Subject: [PATCH 19/28] perf: tune gc by default This is running on several prod site without any noticable problems, so making it default behaviour. Opt out by setting env variable `FRAPPE_TUNE_GC=False` --- .github/helper/install.sh | 2 -- frappe/__init__.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 5a4d341a9b..5cdcbebe1a 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -54,8 +54,6 @@ fi echo "Starting Bench..." -export FRAPPE_TUNE_GC=True - bench start &> ~/frappe-bench/bench_start.log & if [ "$TYPE" == "server" ] diff --git a/frappe/__init__.py b/frappe/__init__.py index 13e9448109..88b995d17b 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -58,7 +58,7 @@ re._MAXCACHE = ( 50 # reduced from default 512 given we are already maintaining this on parent worker ) -_tune_gc = bool(os.environ.get("FRAPPE_TUNE_GC", False)) +_tune_gc = bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True))) if _dev_server: warnings.simplefilter("always", DeprecationWarning) From 265a28e151c3e7301b7221d44b1d79b3ba52cd4c Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jul 2023 16:32:40 +0530 Subject: [PATCH 20/28] perf: preload pydantic --- frappe/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/app.py b/frappe/app.py index 1cbdca1361..ee4ba7d170 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -33,6 +33,8 @@ _sites_path = os.environ.get("SITES_PATH", ".") # If gc.freeze is done then importing modules before forking allows us to share the memory if frappe._tune_gc: + import pydantic + import frappe.boot import frappe.client import frappe.core.doctype.user.user From 35039ca382a68ca705583c66d667d43759d22501 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jul 2023 18:46:24 +0530 Subject: [PATCH 21/28] Revert "test: Bump memory usage treshold post pydantic v2 upgrade" This reverts commit af7032893087db31f5eff6c31c9473561fdd4d0b. --- frappe/core/doctype/rq_job/test_rq_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index cb7faca187..f5d5f89ed4 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -160,7 +160,7 @@ class TestRQJob(FrappeTestCase): # If this starts failing analyze memory usage using memray or some equivalent tool to find # offending imports/function calls. # Refer this PR: https://github.com/frappe/frappe/pull/21467 - LAST_MEASURED_USAGE = 42 + LAST_MEASURED_USAGE = 40 self.assertLessEqual(rss, LAST_MEASURED_USAGE * 1.05, msg) From f5b93cc2a02a9298b7e025a8a107bd204609d95d Mon Sep 17 00:00:00 2001 From: David Arnold Date: Sun, 2 Jul 2023 11:16:53 -0500 Subject: [PATCH 22/28] chore: urlparse already does the parsing for us, no need to diy (#21558) This is a very minor cleanup. So minor, that it hopefully is a nobrainer --- frappe/utils/connections.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/utils/connections.py b/frappe/utils/connections.py index 020bc8b97f..fcca8593ad 100644 --- a/frappe/utils/connections.py +++ b/frappe/utils/connections.py @@ -31,10 +31,9 @@ def check_redis(redis_services=None): config = get_conf() services = redis_services or REDIS_KEYS status = {} - for conn in services: - redis_url = urlparse(config.get(conn)).netloc - redis_host, redis_port = redis_url.split(":") - status[conn] = is_open(redis_host, redis_port) + for srv in services: + url = urlparse(config[srv]) + status[srv] = is_open(url.hostname, url.port) return status From d7990368a8e6145d01b9f38e52ec7c005e292277 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jul 2023 21:47:21 +0530 Subject: [PATCH 23/28] perf: preload more modules (#21557) * perf: preload more modules - bleach is used frequently for sanitization - File gets imported anytime a private file is viewed. Indirect import of PIL is costly in each worker. * test: warm up perf test --- frappe/app.py | 3 +++ frappe/tests/test_perf.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/frappe/app.py b/frappe/app.py index ee4ba7d170..dd7f28f0cc 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -33,14 +33,17 @@ _sites_path = os.environ.get("SITES_PATH", ".") # If gc.freeze is done then importing modules before forking allows us to share the memory if frappe._tune_gc: + import bleach import pydantic import frappe.boot import frappe.client + import frappe.core.doctype.file.file import frappe.core.doctype.user.user import frappe.database.mariadb.database # Load database related utils import frappe.database.query import frappe.desk.desktop # workspace + import frappe.desk.form.save import frappe.model.db_query import frappe.query_builder import frappe.utils.background_jobs # Enqueue is very common diff --git a/frappe/tests/test_perf.py b/frappe/tests/test_perf.py index cc7d0b031d..e7a08299d9 100644 --- a/frappe/tests/test_perf.py +++ b/frappe/tests/test_perf.py @@ -74,6 +74,9 @@ class TestPerformance(FrappeTestCase): def test_get_value_limits(self): # check both dict and list style filters filters = [{"enabled": 1}, [["enabled", "=", 1]]] + + # Warm up code + frappe.db.get_values("User", filters=filters[0], limit=1) for filter in filters: with self.assertRowsRead(1): self.assertEqual(1, len(frappe.db.get_values("User", filters=filter, limit=1))) From 9b6b09c45090e7df1a1ffb379f10a98d6ba17c6d Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Mon, 3 Jul 2023 09:54:27 +0530 Subject: [PATCH 24/28] refactor: Simplify code --- frappe/public/js/frappe/form/formatters.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 81e403169f..09cbc22f54 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -353,9 +353,9 @@ frappe.form.formatters = { const formatted_values = rows.map((row) => { const value = row[link_field.fieldname]; return ( - `` + - frappe.format(value, link_field, options, row) + - `` + ` + ${frappe.format(value, link_field, options, row)} + ` ); }); return formatted_values.join(", "); From b7afc6503294e20199ba5dd5bb6ec3040e48d9d2 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Mon, 3 Jul 2023 10:08:27 +0530 Subject: [PATCH 25/28] style: Fix linter warning --- frappe/public/js/frappe/form/formatters.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 09cbc22f54..4c7eb4cf94 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -352,11 +352,9 @@ frappe.form.formatters = { const link_field = meta.fields.find((df) => df.fieldtype === "Link"); const formatted_values = rows.map((row) => { const value = row[link_field.fieldname]; - return ( - ` - ${frappe.format(value, link_field, options, row)} - ` - ); + return ` + ${frappe.format(value, link_field, options, row)} + `; }); return formatted_values.join(", "); }, From 8fc518baffbebd3fe0cb252fdfea3d448ddd35e3 Mon Sep 17 00:00:00 2001 From: Anh Le Date: Thu, 22 Jun 2023 08:30:05 +0000 Subject: [PATCH 26/28] fix: repeat title value in description of link field. (cherry picked from commit 5957cbcf0908261339e64bf6536e5c18af511ea1) --- frappe/desk/search.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index c4c11558dd..68ea7dff67 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -245,10 +245,6 @@ def search_widget( def get_std_fields_list(meta, key): # get additional search fields sflist = ["name"] - if meta.search_fields: - for d in meta.search_fields.split(","): - if d.strip() not in sflist: - sflist.append(d.strip()) if meta.title_field and meta.title_field not in sflist: sflist.append(meta.title_field) @@ -256,6 +252,11 @@ def get_std_fields_list(meta, key): if key not in sflist: sflist.append(key) + if meta.search_fields: + for d in meta.search_fields.split(","): + if d.strip() not in sflist: + sflist.append(d.strip()) + return sflist From d9167985b13a8d178d14ea46f16653218968774f Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 3 Jul 2023 12:53:20 +0530 Subject: [PATCH 27/28] fix: ensure correct file path when importing package --- frappe/core/doctype/package_import/package_import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/package_import/package_import.py b/frappe/core/doctype/package_import/package_import.py index 4939b357b0..25288603e1 100644 --- a/frappe/core/doctype/package_import/package_import.py +++ b/frappe/core/doctype/package_import/package_import.py @@ -10,6 +10,7 @@ from frappe.desk.form.load import get_attachments from frappe.model.document import Document from frappe.model.sync import get_doc_files from frappe.modules.import_file import import_doc, import_file_by_path +from frappe.utils import get_files_path class PackageImport(Document): @@ -35,7 +36,7 @@ class PackageImport(Document): [ "tar", "xzf", - frappe.get_site_path(attachment.file_url.strip("/")), + get_files_path(attachment.file_name, is_private=attachment.is_private), "-C", frappe.get_site_path("packages"), ] From 00e5dabef144fb916c9daec4591e47302d39d796 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 3 Jul 2023 13:34:44 +0530 Subject: [PATCH 28/28] fix: defer access log inserts in read only mod --- frappe/core/doctype/access_log/access_log.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index c194f5d603..5a6b304e9e 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -59,7 +59,7 @@ def _make_access_log( user = frappe.session.user in_request = frappe.request and frappe.request.method == "GET" - frappe.get_doc( + access_log = frappe.get_doc( { "doctype": "Access Log", "user": user, @@ -72,7 +72,12 @@ def _make_access_log( "filters": cstr(filters) or None, "columns": columns, } - ).db_insert() + ) + + if frappe.flags.read_only: + access_log.deferred_insert() + else: + access_log.db_insert() # `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview` # dont commit in test mode. It must be tempting to put this block along with the in_request in the