diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 5a4d341a9b..5cdcbebe1a 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -54,8 +54,6 @@ fi echo "Starting Bench..." -export FRAPPE_TUNE_GC=True - bench start &> ~/frappe-bench/bench_start.log & if [ "$TYPE" == "server" ] diff --git a/frappe/__init__.py b/frappe/__init__.py index 13e9448109..88b995d17b 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -58,7 +58,7 @@ re._MAXCACHE = ( 50 # reduced from default 512 given we are already maintaining this on parent worker ) -_tune_gc = bool(os.environ.get("FRAPPE_TUNE_GC", False)) +_tune_gc = bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True))) if _dev_server: warnings.simplefilter("always", DeprecationWarning) diff --git a/frappe/app.py b/frappe/app.py index 1cbdca1361..dd7f28f0cc 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -33,12 +33,17 @@ _sites_path = os.environ.get("SITES_PATH", ".") # If gc.freeze is done then importing modules before forking allows us to share the memory if frappe._tune_gc: + import bleach + import pydantic + import frappe.boot import frappe.client + import frappe.core.doctype.file.file import frappe.core.doctype.user.user import frappe.database.mariadb.database # Load database related utils import frappe.database.query import frappe.desk.desktop # workspace + import frappe.desk.form.save import frappe.model.db_query import frappe.query_builder import frappe.utils.background_jobs # Enqueue is very common diff --git a/frappe/auth.py b/frappe/auth.py index 29c3e41694..d1259e1aaf 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -349,8 +349,6 @@ class CookieManager: expires = datetime.datetime.now() + datetime.timedelta(days=3) if frappe.session.sid: self.set_cookie("sid", frappe.session.sid, expires=expires, httponly=True) - if frappe.session.session_country: - self.set_cookie("country", frappe.session.session_country) def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"): if not secure and hasattr(frappe.local, "request"): diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index c194f5d603..5a6b304e9e 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -59,7 +59,7 @@ def _make_access_log( user = frappe.session.user in_request = frappe.request and frappe.request.method == "GET" - frappe.get_doc( + access_log = frappe.get_doc( { "doctype": "Access Log", "user": user, @@ -72,7 +72,12 @@ def _make_access_log( "filters": cstr(filters) or None, "columns": columns, } - ).db_insert() + ) + + if frappe.flags.read_only: + access_log.deferred_insert() + else: + access_log.db_insert() # `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview` # dont commit in test mode. It must be tempting to put this block along with the in_request in the diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 8778826b56..359425d58c 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -724,7 +724,7 @@ def has_permission(doc, ptype=None, user=None): if ptype == "create": return frappe.has_permission("File", "create", user=user) - if not doc.is_private or doc.owner == user or user == "Administrator": + if not doc.is_private or (user != "Guest" and doc.owner == user) or user == "Administrator": return True if doc.attached_to_doctype and doc.attached_to_name: diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 1e7e698062..ef3a7a5f2b 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -29,11 +29,11 @@ test_content1 = "Hello" test_content2 = "Hello World" -def make_test_doc(): +def make_test_doc(ignore_permissions=False): d = frappe.new_doc("ToDo") d.description = "Test" d.assigned_by = frappe.session.user - d.save() + d.save(ignore_permissions) return d.doctype, d.name @@ -785,3 +785,81 @@ class TestFileOptimization(FrappeTestCase): file_content = f.read() self.assertEqual(get_extension("", None, file_content), "jpg") + + +class TestGuestFileAndAttachments(FrappeTestCase): + def setUp(self) -> None: + frappe.db.delete("File", {"is_folder": 0}) + frappe.get_doc( + doctype="DocType", + name="Test For Attachment", + module="Custom", + custom=1, + fields=[ + {"label": "Title", "fieldname": "title", "fieldtype": "Data"}, + {"label": "Attachment", "fieldname": "attachment", "fieldtype": "Attach"}, + ], + ).insert(ignore_if_duplicate=True) + + def tearDown(self) -> None: + frappe.set_user("Administrator") + frappe.db.rollback() + frappe.delete_doc("DocType", "Test For Attachment") + + def test_attach_unattached_guest_file(self): + """Ensure that unattached files are attached on doc update.""" + f = frappe.new_doc( + "File", + file_name="test_private_guest_attachment.txt", + content="Guest Home", + is_private=1, + ).insert(ignore_permissions=True) + + d = frappe.new_doc("Test For Attachment") + d.title = "Test for attachment on update" + d.attachment = f.file_url + d.assigned_by = frappe.session.user + d.save() + + self.assertTrue( + frappe.db.exists( + "File", + { + "file_name": "test_private_guest_attachment.txt", + "file_url": f.file_url, + "attached_to_doctype": "Test For Attachment", + "attached_to_name": d.name, + "attached_to_field": "attachment", + }, + ) + ) + + def test_list_private_guest_single_file(self): + """Ensure that guests are not able to read private standalone guest files.""" + frappe.set_user("Guest") + + file = frappe.new_doc( + "File", + file_name="test_private_guest_single_txt", + content="Private single File", + is_private=1, + ).insert(ignore_permissions=True) + + self.assertFalse(file.is_downloadable()) + + def test_list_private_guest_attachment(self): + """Ensure that guests are not able to read private guest attachments.""" + frappe.set_user("Guest") + + self.attached_to_doctype, self.attached_to_docname = make_test_doc(ignore_permissions=True) + + file = frappe.new_doc( + "File", + file_name="test_private_guest_attachment.txt", + attached_to_doctype=self.attached_to_doctype, + attached_to_name=self.attached_to_docname, + content="Private Attachment", + is_private=1, + ).insert(ignore_permissions=True) + + self.assertFalse(file.is_downloadable()) diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index 1d0d145303..932fce9a03 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -294,10 +294,11 @@ def update_existing_file_docs(doc: "File") -> None: ).run() -def attach_files_to_document(doc: "File", event) -> None: +def attach_files_to_document(doc: "Document", event) -> None: """Runs on on_update hook of all documents. - Goes through every Attach and Attach Image field and attaches - the file url to the document if it is not already attached. + Goes through every file linked with the Attach and Attach Image field and attaches + the file to the document if not already attached. If no file is found, a new file + is created. """ attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}) @@ -320,6 +321,28 @@ def attach_files_to_document(doc: "File", event) -> None: ): return + unattached_file = frappe.db.exists( + "File", + { + "file_url": value, + "attached_to_name": None, + "attached_to_doctype": None, + "attached_to_field": None, + }, + ) + + if unattached_file: + frappe.db.set_value( + "File", + unattached_file, + field={ + "attached_to_name": doc.name, + "attached_to_doctype": doc.doctype, + "attached_to_field": df.fieldname, + }, + ) + return + file: "File" = frappe.get_doc( doctype="File", file_url=value, diff --git a/frappe/core/doctype/package_import/package_import.py b/frappe/core/doctype/package_import/package_import.py index 4939b357b0..25288603e1 100644 --- a/frappe/core/doctype/package_import/package_import.py +++ b/frappe/core/doctype/package_import/package_import.py @@ -10,6 +10,7 @@ from frappe.desk.form.load import get_attachments from frappe.model.document import Document from frappe.model.sync import get_doc_files from frappe.modules.import_file import import_doc, import_file_by_path +from frappe.utils import get_files_path class PackageImport(Document): @@ -35,7 +36,7 @@ class PackageImport(Document): [ "tar", "xzf", - frappe.get_site_path(attachment.file_url.strip("/")), + get_files_path(attachment.file_name, is_private=attachment.is_private), "-C", frappe.get_site_path("packages"), ] diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js index 7d68fd683c..e5c268b24c 100644 --- a/frappe/desk/page/setup_wizard/setup_wizard.js +++ b/frappe/desk/page/setup_wizard/setup_wizard.js @@ -97,10 +97,13 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides { handle_enter_press(e) { if (e.which === frappe.ui.keyCode.ENTER) { - var $target = $(e.target); - if ($target.hasClass("prev-btn")) { + let $target = $(e.target); + if ($target.hasClass("prev-btn") || $target.hasClass("next-btn")) { $target.trigger("click"); } else { + // hitting enter on autocomplete field shouldn't trigger next slide. + if ($target.data().fieldtype == "Autocomplete") return; + this.container.find(".next-btn").trigger("click"); e.preventDefault(); } @@ -545,15 +548,19 @@ frappe.setup.utils = { slide.get_input("timezone").empty().add_options(data.all_timezones); - // set values if present - if (frappe.wizard.values.country) { - country_field.set_input(frappe.wizard.values.country); - } else if (data.default_country) { - country_field.set_input(data.default_country); - } - slide.get_field("currency").set_input(frappe.wizard.values.currency); slide.get_field("timezone").set_input(frappe.wizard.values.timezone); + + // set values if present + let country = + frappe.wizard.values.country || + data.default_country || + guess_country(frappe.setup.data.regional_data.country_info); + + if (country) { + country_field.set_input(country); + $(country_field.input).change(); + } }, bind_language_events: function (slide) { @@ -630,3 +637,16 @@ frappe.setup.utils = { }); }, }; + +function guess_country(country_info) { + try { + const system_timezone = Intl.DateTimeFormat().resolvedOptions().timeZone; + + for ([country, info] of Object.entries(country_info)) { + let possible_timezones = (info.timezones || []).filter((t) => t == system_timezone); + if (possible_timezones.length) return country; + } + } catch (e) { + console.log("Could not guess country", e); + } +} diff --git a/frappe/desk/search.py b/frappe/desk/search.py index c4c11558dd..68ea7dff67 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -245,10 +245,6 @@ def search_widget( def get_std_fields_list(meta, key): # get additional search fields sflist = ["name"] - if meta.search_fields: - for d in meta.search_fields.split(","): - if d.strip() not in sflist: - sflist.append(d.strip()) if meta.title_field and meta.title_field not in sflist: sflist.append(meta.title_field) @@ -256,6 +252,11 @@ def get_std_fields_list(meta, key): if key not in sflist: sflist.append(key) + if meta.search_fields: + for d in meta.search_fields.split(","): + if d.strip() not in sflist: + sflist.append(d.strip()) + return sflist diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 500a126f72..cae5f76b3d 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -149,7 +149,7 @@ def flush(from_test=False): now=from_test, job_id=f"email_queue_sendmail_{row.name}", queue="short", - dedupicate=True, + deduplicate=True, ) except Exception: frappe.get_doc("Email Queue", row.name).log_error() diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index beddbf512d..4c7eb4cf94 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -352,7 +352,9 @@ frappe.form.formatters = { const link_field = meta.fields.find((df) => df.fieldtype === "Link"); const formatted_values = rows.map((row) => { const value = row[link_field.fieldname]; - return frappe.format(value, link_field, options, row); + return ` + ${frappe.format(value, link_field, options, row)} + `; }); return formatted_values.join(", "); }, diff --git a/frappe/sessions.py b/frappe/sessions.py index 2709de8fab..b0eb7e7353 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -236,9 +236,6 @@ class Session: "session_expiry": get_expiry_period(), "full_name": self.full_name, "user_type": self.user_type, - "session_country": get_geo_ip_country(frappe.local.request_ip) - if frappe.local.request_ip - else None, } ) diff --git a/frappe/tests/test_perf.py b/frappe/tests/test_perf.py index cc7d0b031d..e7a08299d9 100644 --- a/frappe/tests/test_perf.py +++ b/frappe/tests/test_perf.py @@ -74,6 +74,9 @@ class TestPerformance(FrappeTestCase): def test_get_value_limits(self): # check both dict and list style filters filters = [{"enabled": 1}, [["enabled", "=", 1]]] + + # Warm up code + frappe.db.get_values("User", filters=filters[0], limit=1) for filter in filters: with self.assertRowsRead(1): self.assertEqual(1, len(frappe.db.get_values("User", filters=filter, limit=1))) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 9997522692..59e942ef42 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1163,3 +1163,44 @@ class TestRounding(FrappeTestCase): def test_default_rounding(self): self.assertEqual(frappe.get_system_settings("rounding_method"), "Banker's Rounding") + + +class TestTypingValidations(FrappeTestCase): + def test_validate_argument_types(self): + from frappe.core.doctype.doctype.doctype import DocType + from frappe.utils.typing_validations import FrappeTypeError, validate_argument_types + + @validate_argument_types + def test_simple_types(a: int, b: float, c: bool): + return a, b, c + + @validate_argument_types + def test_sequence(a: str, b: list[dict] | None = None, c: dict[str, int] | None = None): + return a, b, c + + @validate_argument_types + def test_doctypes(a: DocType | dict): + return a + + self.assertEqual(test_simple_types(True, 2.0, True), (1, 2.0, True)) + self.assertEqual(test_simple_types(1, 2, 1), (1, 2.0, True)) + self.assertEqual(test_simple_types(1.0, 2, 1), (1, 2.0, True)) + self.assertEqual(test_simple_types(1, 2, "1"), (1, 2.0, True)) + with self.assertRaises(FrappeTypeError): + test_simple_types(1, 2, "a") + with self.assertRaises(FrappeTypeError): + test_simple_types(1, 2, None) + + self.assertEqual(test_sequence("a", [{"a": 1}], {"a": 1}), ("a", [{"a": 1}], {"a": 1})) + self.assertEqual(test_sequence("a", None, None), ("a", None, None)) + self.assertEqual(test_sequence("a", [{"a": 1}], None), ("a", [{"a": 1}], None)) + self.assertEqual(test_sequence("a", None, {"a": 1}), ("a", None, {"a": 1})) + self.assertEqual(test_sequence("a", [{"a": 1}], {"a": "1.0"}), ("a", [{"a": 1}], {"a": 1})) + with self.assertRaises(FrappeTypeError): + test_sequence("a", [{"a": 1}], True) + + doctype = frappe.get_last_doc("DocType") + self.assertEqual(test_doctypes(doctype), doctype) + self.assertEqual(test_doctypes(doctype.as_dict()), doctype.as_dict()) + with self.assertRaises(FrappeTypeError): + test_doctypes("a") diff --git a/frappe/utils/connections.py b/frappe/utils/connections.py index 020bc8b97f..fcca8593ad 100644 --- a/frappe/utils/connections.py +++ b/frappe/utils/connections.py @@ -31,10 +31,9 @@ def check_redis(redis_services=None): config = get_conf() services = redis_services or REDIS_KEYS status = {} - for conn in services: - redis_url = urlparse(config.get(conn)).netloc - redis_host, redis_port = redis_url.split(":") - status[conn] = is_open(redis_host, redis_port) + for srv in services: + url = urlparse(config[srv]) + status[srv] = is_open(url.hostname, url.port) return status diff --git a/frappe/utils/typing_validations.py b/frappe/utils/typing_validations.py index e7ebcfbdff..91a318eae4 100644 --- a/frappe/utils/typing_validations.py +++ b/frappe/utils/typing_validations.py @@ -1,11 +1,7 @@ from functools import lru_cache, wraps from inspect import _empty, isclass, signature from types import EllipsisType -from typing import Any, Callable, ForwardRef, TypeVar, Union - -from pydantic.config import BaseConfig -from pydantic.error_wrappers import ValidationError as PyValidationError -from pydantic.tools import NameFactory, _generate_parsing_type_name +from typing import Callable, ForwardRef, TypeVar, Union from frappe.exceptions import FrappeTypeError @@ -69,29 +65,10 @@ def raise_type_error( @lru_cache(maxsize=2048) -def _get_parsing_type( - type_: Any, *, type_name: NameFactory | None = None, config: type[BaseConfig] = None -) -> Any: - # Note: this is a copy of pydantic.tools._get_parsing_type with the addition of allowing a config argument - from pydantic.main import create_model +def TypeAdapter(type_): + from pydantic import TypeAdapter as PyTypeAdapter - if type_name is None: - type_name = _generate_parsing_type_name - if not isinstance(type_name, str): - type_name = type_name(type_) - return create_model(type_name, __root__=(type_, ...), __config__=config) - - -def parse_obj_as( - type_: type[T], - obj: Any, - *, - type_name: NameFactory | None = None, - config: type[BaseConfig] | None = None, -) -> T: - # Note: This is a copy of pydantic.tools.parse_obj_as with the addition of allowing a config argument - model_type = _get_parsing_type(type_, type_name=type_name, config=config) # type: ignore[arg-type] - return model_type(__root__=obj).__root__ + return PyTypeAdapter(type_, config=FrappePydanticConfig) def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): @@ -103,6 +80,8 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): if not (args or kwargs) or not func.__annotations__: return args, kwargs + from pydantic import ValidationError as PyValidationError + annotations = func.__annotations__ new_args, new_kwargs = list(args), kwargs @@ -157,9 +136,7 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): # validate the type set using pydantic - raise a TypeError if Validation is raised or Ellipsis is returned try: - current_arg_value_after = parse_obj_as( - current_arg_type, current_arg_value, type_name=current_arg, config=FrappePydanticConfig - ) + current_arg_value_after = TypeAdapter(current_arg_type).validate_python(current_arg_value) except (TypeError, PyValidationError) as e: raise_type_error(current_arg, current_arg_type, current_arg_value, current_exception=e) diff --git a/pyproject.toml b/pyproject.toml index f2ce88f294..17a75f3c0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "psycopg2-binary~=2.9.1", "pyOpenSSL~=23.2.0", "pycryptodome~=3.18.0", - "pydantic~=1.10.8", + "pydantic==2.0", "pyotp~=2.8.0", "python-dateutil~=2.8.2", "pytz==2023.3",