diff --git a/frappe/__init__.py b/frappe/__init__.py index e5a0b9c4aa..ff2a663d50 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -16,6 +16,7 @@ import inspect import json import os import re +import unicodedata import warnings from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, TypeAlias, overload @@ -190,7 +191,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False) local.error_log = [] local.message_log = [] local.debug_log = [] - local.realtime_log = [] local.flags = _dict( { "currently_saving": [], @@ -207,9 +207,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False) "read_only": False, } ) - local.rollback_observers = [] local.locked_documents = [] - local.before_commit = [] local.test_objects = {} local.site = site @@ -233,7 +231,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False) local.role_permissions = {} local.valid_columns = {} local.new_doc_templates = {} - local.link_count = {} local.jenv = None local.jloader = None @@ -879,6 +876,7 @@ def clear_cache(user: str | None = None, doctype: str | None = None): :param doctype: If doctype is given, only DocType cache is cleared.""" import frappe.cache_manager import frappe.utils.caching + from frappe.website.router import clear_routing_cache if doctype: frappe.cache_manager.clear_doctype_cache(doctype) @@ -907,6 +905,8 @@ def clear_cache(user: str | None = None, doctype: str | None = None): if hasattr(local, "website_settings"): del local.website_settings + clear_routing_cache() + def only_has_select_perm(doctype, user=None, ignore_permissions=False): if ignore_permissions: @@ -1079,7 +1079,7 @@ def set_value(doctype, docname, fieldname, value=None): def get_cached_doc(*args, **kwargs) -> "Document": - if (key := can_cache_doc(args)) and (doc := cache().hget("document_cache", key)): + if (key := can_cache_doc(args)) and (doc := cache().get_value(key)): return doc # Not found in cache, fetch from DB @@ -1095,7 +1095,7 @@ def get_cached_doc(*args, **kwargs) -> "Document": def _set_document_in_cache(key: str, doc: "Document") -> None: - cache().hset("document_cache", key, doc) + cache().set_value(key, doc) def can_cache_doc(args) -> str | None: @@ -1116,12 +1116,20 @@ def can_cache_doc(args) -> str | None: def get_document_cache_key(doctype: str, name: str): - return f"{doctype}::{name}" + return f"document_cache::{doctype}::{name}" -def clear_document_cache(doctype, name): - cache().hdel("last_modified", doctype) - cache().hdel("document_cache", get_document_cache_key(doctype, name)) +def clear_document_cache(doctype: str, name: str | None = None) -> None: + def clear_in_redis(): + if name is not None: + cache().delete_value(get_document_cache_key(doctype, name)) + else: + cache().delete_keys(get_document_cache_key(doctype, "")) + + clear_in_redis() + if hasattr(db, "after_commit"): + db.after_commit.add(clear_in_redis) + db.after_rollback.add(clear_in_redis) if doctype == "System Settings" and hasattr(local, "system_settings"): delattr(local, "system_settings") @@ -1206,7 +1214,7 @@ def get_doc(*args, **kwargs): doc = frappe.model.document.get_doc(*args, **kwargs) # Replace cache if stale one exists - if (key := can_cache_doc(args)) and cache().hexists("document_cache", key): + if (key := can_cache_doc(args)) and cache().exists(key): _set_document_in_cache(key, doc) return doc @@ -2271,6 +2279,7 @@ def bold(text): def safe_eval(code, eval_globals=None, eval_locals=None): """A safer `eval`""" whitelisted_globals = {"int": int, "float": float, "long": int, "round": round} + code = unicodedata.normalize("NFKC", code) UNSAFE_ATTRIBUTES = { # Generator Attributes diff --git a/frappe/app.py b/frappe/app.py index fab8facd3f..55855efaf9 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -19,7 +19,6 @@ import frappe.recorder import frappe.utils.response from frappe import _ from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest -from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request from frappe.middlewares import StaticDataMiddleware from frappe.utils import cint, get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot @@ -351,8 +350,6 @@ def sync_database(rollback: bool) -> bool: frappe.db.commit() rollback = False - update_comments_in_parent_after_request() - return rollback diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 12e829ff09..9c1754148a 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -123,12 +123,21 @@ def clear_defaults_cache(user=None): def clear_doctype_cache(doctype=None): clear_controller_cache(doctype) + + _clear_doctype_cache_form_redis() + if hasattr(frappe.db, "after_commit"): + frappe.db.after_commit.add(_clear_doctype_cache_form_redis) + frappe.db.after_rollback.add(_clear_doctype_cache_form_redis) + + +def _clear_doctype_cache_form_redis(doctype: str | None = None): cache = frappe.cache() - for key in ("is_table", "doctype_modules", "document_cache"): + for key in ("is_table", "doctype_modules"): cache.delete_value(key) def clear_single(dt): + frappe.clear_document_cache(dt) for name in doctype_cache_keys: cache.hdel(name, dt) @@ -155,6 +164,7 @@ def clear_doctype_cache(doctype=None): # clear all for name in doctype_cache_keys: cache.delete_value(name) + cache.delete_keys("document_cache::") def clear_controller_cache(doctype=None): diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 03374986d4..e44009a886 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -623,7 +623,7 @@ frappe.db.connect() def _console_cleanup(): - # Execute rollback_observers on console close + # Execute after_rollback on console close frappe.db.rollback() frappe.destroy() diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index dff13e1170..c86c7811ad 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -152,14 +152,9 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): except Exception as e: if frappe.db.is_column_missing(e) and getattr(frappe.local, "request", None): - # missing column and in request, add column and update after commit - frappe.local._comments = getattr(frappe.local, "_comments", []) + [ - (reference_doctype, reference_name, _comments) - ] - + pass elif frappe.db.is_data_too_long(e): raise frappe.DataTooLongException - else: raise else: @@ -169,13 +164,3 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): # Clear route cache if route := frappe.get_cached_value(reference_doctype, reference_name, "route"): clear_cache(route) - - -def update_comments_in_parent_after_request(): - """update _comments in parent if _comments column is missing""" - if hasattr(frappe.local, "_comments"): - for (reference_doctype, reference_name, _comments) in frappe.local._comments: - add_column(reference_doctype, "_comments", "Text") - update_comments_in_parent(reference_doctype, reference_name, _comments) - - frappe.db.commit() diff --git a/frappe/core/doctype/doctype/doctype_list.js b/frappe/core/doctype/doctype/doctype_list.js index c66edf1e21..f4811fa01d 100644 --- a/frappe/core/doctype/doctype/doctype_list.js +++ b/frappe/core/doctype/doctype/doctype_list.js @@ -6,16 +6,16 @@ frappe.listview_settings["DocType"] = { setup_select_primary_button: function (me) { let actions = [ - { - label: __("Add DocType"), - description: __("Create a new DocType"), - action: () => frappe.new_doc("DocType"), - }, { label: __("Add DocType (Form Builder)"), description: __("Use the form builder to create a new DocType"), action: () => frappe.set_route("form-builder", "new-doctype"), }, + { + label: __("Add DocType"), + description: __("Create a new DocType"), + action: () => frappe.new_doc("DocType"), + }, ]; frappe.utils.add_select_group_button( diff --git a/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py index 98ce9e738b..bcd3197112 100644 --- a/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py +++ b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py @@ -2,6 +2,7 @@ # See license.txt import frappe +from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.core.doctype.document_naming_settings.document_naming_settings import ( DocumentNamingSettings, ) @@ -11,6 +12,25 @@ from frappe.utils import cint class TestNamingSeries(FrappeTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.ns_doctype = ( + new_doctype( + fields=[ + { + "label": "Series", + "fieldname": "naming_series", + "fieldtype": "Select", + "options": f"\n{frappe.generate_hash()}-.###", + } + ], + autoname="naming_series:", + ) + .insert() + .name + ) + def setUp(self): self.dns: DocumentNamingSettings = frappe.get_doc("Document Naming Settings") @@ -23,7 +43,7 @@ class TestNamingSeries(FrappeTestCase): return VALID_SERIES + exisiting_series def test_naming_preview(self): - self.dns.transaction_type = "Webhook" + self.dns.transaction_type = self.ns_doctype self.dns.try_naming_series = "AXBZ.####" serieses = self.dns.preview_series().split("\n") @@ -35,23 +55,22 @@ class TestNamingSeries(FrappeTestCase): def test_get_transactions(self): naming_info = self.dns.get_transactions_and_prefixes() - self.assertIn("Webhook", naming_info["transactions"]) + self.assertIn(self.ns_doctype, naming_info["transactions"]) - existing_naming_series = frappe.get_meta("Webhook").get_field("naming_series").options + existing_naming_series = frappe.get_meta(self.ns_doctype).get_field("naming_series").options for series in existing_naming_series.split("\n"): self.assertIn(NamingSeries(series).get_prefix(), naming_info["prefixes"]) def test_default_naming_series(self): - self.assertIn("HOOK", get_default_naming_series("Webhook")) self.assertIsNone(get_default_naming_series("DocType")) def test_updates_naming_options(self): - self.dns.transaction_type = "Webhook" + self.dns.transaction_type = self.ns_doctype test_series = "KOOHBEW.###" self.dns.naming_series_options = self.dns.get_options() + "\n" + test_series self.dns.update_series() - self.assertIn(test_series, frappe.get_meta("Webhook").get_naming_series_options()) + self.assertIn(test_series, frappe.get_meta(self.ns_doctype).get_naming_series_options()) def test_update_series_counter(self): for series in self.get_valid_serieses(): diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 3728bd0af0..c4cefc7271 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -69,7 +69,7 @@ class File(Document): else: self.save_file(content=self.get_content()) self.flags.new_file = True - frappe.local.rollback_observers.append(self) + frappe.db.after_rollback.add(self.on_rollback) def after_insert(self): if not self.is_folder: @@ -121,10 +121,16 @@ class File(Document): self.add_comment_in_reference_doc("Attachment Removed", _("Removed {0}").format(self.file_name)) def on_rollback(self): + rollback_flags = ("new_file", "original_content", "original_path") + + def pop_rollback_flags(): + for flag in rollback_flags: + self.flags.pop(flag, None) + # following condition is only executed when an insert has been rolledback if self.flags.new_file: self._delete_file_on_disk() - self.flags.pop("new_file") + pop_rollback_flags() return # if original_content flag is set, this rollback should revert the file to its original state @@ -139,14 +145,14 @@ class File(Document): with open(file_path, mode) as f: f.write(self.flags.original_content) os.fsync(f.fileno()) - self.flags.pop("original_content") + pop_rollback_flags() # used in case file path (File.file_url) has been changed if self.flags.original_path: target = self.flags.original_path["old"] source = self.flags.original_path["new"] shutil.move(source, target) - self.flags.pop("original_path") + pop_rollback_flags() def get_name_based_on_parent_folder(self) -> str | None: if self.folder: @@ -218,7 +224,7 @@ class File(Document): # Uses os.rename which is an atomic operation shutil.move(source, target) self.flags.original_path = {"old": source, "new": target} - frappe.local.rollback_observers.append(self) + frappe.db.after_rollback.add(self.on_rollback) self.file_url = updated_file_url update_existing_file_docs(self) @@ -520,7 +526,7 @@ class File(Document): f.write(self._content) os.fsync(f.fileno()) - frappe.local.rollback_observers.append(self) + frappe.db.after_rollback.add(self.on_rollback) return file_path diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 51e065f710..bbe8bb6d1a 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -17,7 +17,7 @@ from frappe.core.api.file import ( move_file, unzip_file, ) -from frappe.core.doctype.file.utils import get_extension +from frappe.core.doctype.file.utils import delete_file, get_extension from frappe.exceptions import ValidationError from frappe.tests.utils import FrappeTestCase from frappe.utils import get_files_path @@ -77,6 +77,16 @@ class TestSimpleFile(FrappeTestCase): self.assertEqual(content, self.test_content) +class TestFSRollbacks(FrappeTestCase): + def test_rollback_from_file_system(self): + file_name = content = frappe.generate_hash() + file = frappe.new_doc("File", file_name=file_name, content=content).insert() + self.assertTrue(file.exists_on_disk()) + + frappe.db.rollback() + self.assertFalse(file.exists_on_disk()) + + class TestBase64File(FrappeTestCase): def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index 265583fe83..09a90f7445 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -11,7 +11,7 @@ import frappe from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs, stop_job from frappe.tests.utils import FrappeTestCase, timeout from frappe.utils import cstr, execute_in_shell -from frappe.utils.background_jobs import is_job_enqueued +from frappe.utils.background_jobs import get_job_status, is_job_enqueued class TestRQJob(FrappeTestCase): @@ -104,6 +104,26 @@ class TestRQJob(FrappeTestCase): self.check_status(job, "finished") self.assertFalse(is_job_enqueued(job_id)) + @timeout(20) + def test_enqueue_after_commit(self): + job_id = frappe.generate_hash() + + frappe.enqueue(self.BG_JOB, enqueue_after_commit=True, job_id=job_id) + self.assertIsNone(get_job_status(job_id)) + + frappe.db.commit() + self.assertIsNotNone(get_job_status(job_id)) + + job_id = frappe.generate_hash() + frappe.enqueue(self.BG_JOB, enqueue_after_commit=True, job_id=job_id) + self.assertIsNone(get_job_status(job_id)) + + frappe.db.rollback() + self.assertIsNone(get_job_status(job_id)) + + frappe.db.commit() + self.assertIsNone(get_job_status(job_id)) + def test_func(fail=False, sleep=0): if fail: diff --git a/frappe/core/doctype/user/user.json b/frappe/core/doctype/user/user.json index 654f20936e..0396776183 100644 --- a/frappe/core/doctype/user/user.json +++ b/frappe/core/doctype/user/user.json @@ -212,6 +212,7 @@ "read_only": 1 }, { + "allow_in_quick_entry": 1, "fieldname": "role_profile_name", "fieldtype": "Link", "label": "Role Profile", @@ -761,7 +762,7 @@ "link_fieldname": "user" } ], - "modified": "2023-05-24 15:20:06.434506", + "modified": "2023-06-05 17:26:04.127555", "modified_by": "Administrator", "module": "Core", "name": "User", diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 94ea8b16a0..81d9715c32 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -75,6 +75,7 @@ class User(Document): self.validate_email_type(self.email) self.validate_email_type(self.name) self.add_system_manager_role() + self.check_roles_added() self.set_system_user() self.set_full_name() self.check_enable_disable() @@ -673,6 +674,21 @@ class User(Document): if not self.time_zone: self.time_zone = get_system_timezone() + def check_roles_added(self): + if self.user_type != "System User" or self.roles or not self.is_new(): + return + + frappe.msgprint( + _("Newly created user {0} has no roles enabled.").format(frappe.bold(self.name)), + title=_("No Roles Specified"), + indicator="orange", + primary_action={ + "label": _("Add Roles"), + "client_action": "frappe.set_route", + "args": ["Form", self.doctype, self.name], + }, + ) + @frappe.whitelist() def get_timezones(): diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 9e6b8990d5..9aa61869d3 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -192,7 +192,9 @@ class CustomizeForm(Document): if self.flags.rebuild_doctype_for_global_search: frappe.enqueue( - "frappe.utils.global_search.rebuild_for_doctype", now=True, doctype=self.doc_type + "frappe.utils.global_search.rebuild_for_doctype", + doctype=self.doc_type, + enqueue_after_commit=True, ) def set_property_setters(self): diff --git a/frappe/database/database.py b/frappe/database/database.py index 728d1e9584..2bac5a1ffc 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -29,8 +29,8 @@ from frappe.database.utils import ( is_query_type, ) from frappe.exceptions import DoesNotExistError, ImplicitCommitError -from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count +from frappe.utils import CallbackManager from frappe.utils import cast as cast_fieldtype from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool from frappe.utils.deprecations import deprecated, deprecation_warning @@ -107,6 +107,12 @@ class Database: self.value_cache = {} self.logger = frappe.logger("database") self.logger.setLevel("WARNING") + + self.before_commit = CallbackManager() + self.after_commit = CallbackManager() + self.before_rollback = CallbackManager() + self.after_rollback = CallbackManager() + # self.db_type: str # self.last_query (lazy) attribute of last sql query executed @@ -118,7 +124,6 @@ class Database: self.cur_db_name = self.user self._conn = self.get_connection() self._cursor = self._conn.cursor() - frappe.local.rollback_observers = [] try: if execution_timeout := get_query_execution_timeout(): @@ -915,10 +920,8 @@ class Database: if isinstance(dn, str): frappe.clear_document_cache(dt, dn) else: - # TODO: Fix this; doesn't work rn - gavin@frappe.io - # frappe.cache().hdel_keys(dt, "document_cache") - # Workaround: clear all document caches - frappe.cache().delete_value("document_cache") + # No way to guess which documents are modified, clear all of them + frappe.clear_document_cache(dt) for column, value in to_update.items(): query = query.set(column, value) @@ -970,26 +973,30 @@ class Database: def commit(self): """Commit current transaction. Calls SQL `COMMIT`.""" - for method in frappe.local.before_commit: - frappe.call(method[0], *(method[1] or []), **(method[2] or {})) + self.before_rollback.reset() + self.after_rollback.reset() + + self.before_commit.run() self.sql("commit") self.begin() # explicitly start a new transaction - frappe.local.rollback_observers = [] - self.flush_realtime_log() - enqueue_jobs_after_commit() - flush_local_link_count() + self.after_commit.run() - def add_before_commit(self, method, args=None, kwargs=None): - frappe.local.before_commit.append([method, args, kwargs]) + def rollback(self, *, save_point=None): + """`ROLLBACK` current transaction. Optionally rollback to a known save_point.""" + if save_point: + self.sql(f"rollback to savepoint {save_point}") + else: + self.before_commit.reset() + self.after_commit.reset() - @staticmethod - def flush_realtime_log(): - for args in frappe.local.realtime_log: - frappe.realtime.emit_via_redis(*args) + self.before_rollback.run() - frappe.local.realtime_log = [] + self.sql("rollback") + self.begin() + + self.after_rollback.run() def savepoint(self, save_point): """Savepoints work as a nested transaction. @@ -1004,21 +1011,6 @@ class Database: def release_savepoint(self, save_point): self.sql(f"release savepoint {save_point}") - def rollback(self, *, save_point=None): - """`ROLLBACK` current transaction. Optionally rollback to a known save_point.""" - if save_point: - self.sql(f"rollback to savepoint {save_point}") - else: - self.sql("rollback") - self.begin() - for obj in dict.fromkeys(frappe.local.rollback_observers): - if hasattr(obj, "on_rollback"): - obj.on_rollback() - frappe.local.rollback_observers = [] - - frappe.local.realtime_log = [] - frappe.flags.enqueue_after_commit = [] - def field_exists(self, dt, fn): """Return true of field exists.""" return self.exists("DocField", {"fieldname": fn, "parent": dt}) @@ -1304,28 +1296,6 @@ class Database: raise NotImplementedError -def enqueue_jobs_after_commit(): - from frappe.utils.background_jobs import ( - RQ_JOB_FAILURE_TTL, - RQ_RESULTS_TTL, - execute_job, - get_queue, - ) - - if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0: - for job in frappe.flags.enqueue_after_commit: - q = get_queue(job.get("queue"), is_async=job.get("is_async")) - q.enqueue_call( - execute_job, - timeout=job.get("timeout"), - kwargs=job.get("queue_args"), - failure_ttl=frappe.conf.get("rq_job_failure_ttl") or RQ_JOB_FAILURE_TTL, - result_ttl=frappe.conf.get("rq_results_ttl") or RQ_RESULTS_TTL, - job_id=job.get("job_id"), - ) - frappe.flags.enqueue_after_commit = [] - - @contextmanager def savepoint(catch: type | tuple[type, ...] = Exception): """Wrapper for wrapping blocks of DB operations in a savepoint. diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 8a65cc1619..390f519367 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -95,7 +95,7 @@ frappe.ui.form.on("Form Tour", { }, }); -add_custom_button = (frm) => { +let add_custom_button = (frm) => { if (frm.doc.ui_tour) { frm.add_custom_button(__("Reset"), function () { frappe.confirm( diff --git a/frappe/desk/doctype/list_view_settings/list_view_settings.py b/frappe/desk/doctype/list_view_settings/list_view_settings.py index 36ebce34d5..e3b6a60a42 100644 --- a/frappe/desk/doctype/list_view_settings/list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/list_view_settings.py @@ -6,8 +6,7 @@ from frappe.model.document import Document class ListViewSettings(Document): - def on_update(self): - frappe.clear_document_cache(self.doctype, self.name) + pass @frappe.whitelist() diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py index 7235447662..2edf2fcf5c 100644 --- a/frappe/integrations/doctype/webhook/test_webhook.py +++ b/frappe/integrations/doctype/webhook/test_webhook.py @@ -14,7 +14,10 @@ from frappe.tests.utils import FrappeTestCase @contextmanager def get_test_webhook(config): - wh = frappe.get_doc(config).insert() + wh = frappe.get_doc(config) + if not wh.name: + wh.name = frappe.generate_hash() + wh.insert() wh.reload() try: yield wh @@ -37,6 +40,7 @@ class TestWebhook(FrappeTestCase): def create_sample_webhooks(cls): samples_webhooks_data = [ { + "name": frappe.generate_hash(), "webhook_doctype": "User", "webhook_docevent": "after_insert", "request_url": "https://httpbin.org/post", @@ -44,6 +48,7 @@ class TestWebhook(FrappeTestCase): "enabled": True, }, { + "name": frappe.generate_hash(), "webhook_doctype": "User", "webhook_docevent": "after_insert", "request_url": "https://httpbin.org/post", diff --git a/frappe/integrations/doctype/webhook/webhook.json b/frappe/integrations/doctype/webhook/webhook.json index cfb2a2e01c..404e0be944 100644 --- a/frappe/integrations/doctype/webhook/webhook.json +++ b/frappe/integrations/doctype/webhook/webhook.json @@ -1,13 +1,12 @@ { "actions": [], - "autoname": "naming_series:", + "autoname": "prompt", "creation": "2017-09-08 16:16:13.060641", "doctype": "DocType", "editable_grid": 1, "engine": "InnoDB", "field_order": [ "sb_doc_events", - "naming_series", "webhook_doctype", "cb_doc_events", "webhook_docevent", @@ -46,6 +45,7 @@ { "fieldname": "webhook_doctype", "fieldtype": "Link", + "in_list_view": 1, "label": "DocType", "options": "DocType", "reqd": 1, @@ -136,12 +136,6 @@ "label": "JSON Request Body", "options": "JSON" }, - { - "fieldname": "naming_series", - "fieldtype": "Select", - "label": "Naming Series", - "options": "\nHOOK-.####" - }, { "fieldname": "sb_security", "fieldtype": "Section Break", @@ -218,11 +212,11 @@ "link_fieldname": "webhook" } ], - "modified": "2023-05-22 16:30:10.740512", + "modified": "2023-06-02 17:25:12.598232", "modified_by": "Administrator", "module": "Integrations", "name": "Webhook", - "naming_rule": "By \"Naming Series\" field", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -241,6 +235,5 @@ "sort_field": "modified", "sort_order": "DESC", "states": [], - "title_field": "webhook_doctype", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/model/document.py b/frappe/model/document.py index 75c3a005c9..f944b28a49 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1200,7 +1200,6 @@ class Document(BaseDocument): if notify: self.notify_update() - self.clear_cache() if commit: frappe.db.commit() diff --git a/frappe/model/utils/link_count.py b/frappe/model/utils/link_count.py index 9a7694b9f8..49ed0d5a6c 100644 --- a/frappe/model/utils/link_count.py +++ b/frappe/model/utils/link_count.py @@ -1,6 +1,8 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from collections import defaultdict + import frappe ignore_doctypes = ("DocType", "Print Format", "Role", "Module Def", "Communication", "ToDo") @@ -8,29 +10,29 @@ ignore_doctypes = ("DocType", "Print Format", "Role", "Module Def", "Communicati def notify_link_count(doctype, name): """updates link count for given document""" - if hasattr(frappe.local, "link_count"): - if (doctype, name) in frappe.local.link_count: - frappe.local.link_count[(doctype, name)] += 1 - else: - frappe.local.link_count[(doctype, name)] = 1 + if not hasattr(frappe.local, "_link_count"): + frappe.local._link_count = defaultdict(int) + frappe.db.after_commit.add(flush_local_link_count) + + frappe.local._link_count[(doctype, name)] += 1 def flush_local_link_count(): """flush from local before ending request""" - if not getattr(frappe.local, "link_count", None): + new_links = getattr(frappe.local, "_link_count", None) + if not new_links: return - link_count = frappe.cache().get_value("_link_count") - if not link_count: - link_count = {} + link_count = frappe.cache().get_value("_link_count") or {} - for key, value in frappe.local.link_count.items(): - if key in link_count: - link_count[key] += frappe.local.link_count[key] - else: - link_count[key] = frappe.local.link_count[key] + for key, value in new_links.items(): + if key in link_count: + link_count[key] += value + else: + link_count[key] = value frappe.cache().set_value("_link_count", link_count) + new_links.clear() def update_link_count(): @@ -38,14 +40,12 @@ def update_link_count(): link_count = frappe.cache().get_value("_link_count") if link_count: - for key, count in link_count.items(): - if key[0] not in ignore_doctypes: + for (doctype, name), count in link_count.items(): + if doctype not in ignore_doctypes: try: - frappe.db.sql( - f"update `tab{key[0]}` set idx = idx + {count} where name=%s", - key[1], - auto_commit=1, - ) + table = frappe.qb.DocType(doctype) + frappe.qb.update(table).set(table.idx, table.idx + count).where(table.name == name).run() + frappe.db.commit() except Exception as e: if not frappe.db.is_table_missing(e): # table not found, single raise e diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js index aa75a2282b..ada729fd66 100644 --- a/frappe/public/js/frappe/form/controls/geolocation.js +++ b/frappe/public/js/frappe/form/controls/geolocation.js @@ -5,26 +5,28 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f async make() { super.make(); + $(this.input_area).addClass("hidden"); } - make_wrapper() { + set_disp_area(value) { // Create the elements for map area - super.make_wrapper(); + if (!this.disp_area) { + return; + } - let $input_wrapper = this.$wrapper.find(".control-input-wrapper"); this.map_id = frappe.dom.get_unique_id(); this.map_area = $( `
Multiple webforms can be created for a single doctype. Write a condition specific to this webform to display correct record after submission.
For Example:
\nIf you create a separate webform every year to capture feedback from employees add a \n field named year in doctype and add a condition doc.year==\"2023\"
\n" } ], "has_web_view": 1, "icon": "icon-edit", "is_published_field": "published", "links": [], - "modified": "2023-04-20 17:24:42.657731", + "modified": "2023-06-03 19:18:56.760479", "modified_by": "Administrator", "module": "Website", "name": "Web Form", diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 3e2705bdbe..fd9949c45f 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -12,6 +12,7 @@ from frappe.desk.form.meta import get_code_files_via_hooks from frappe.modules.utils import export_module_json, get_doc_module from frappe.rate_limiter import rate_limit from frappe.utils import cstr, dict_with_keys, strip_html +from frappe.utils.caching import redis_cache from frappe.website.utils import get_boot_data, get_comment_list, get_sidebar_items from frappe.website.website_generator import WebsiteGenerator @@ -19,9 +20,6 @@ from frappe.website.website_generator import WebsiteGenerator class WebForm(WebsiteGenerator): website = frappe._dict(no_cache=1) - def onload(self): - super().onload() - def validate(self): super().validate() @@ -153,10 +151,16 @@ def get_context(context): and not frappe.form_dict.name and not frappe.form_dict.is_list ): - name = frappe.db.get_value(self.doc_type, {"owner": frappe.session.user}, "name") - if name: - context.in_view_mode = True - frappe.redirect(f"/{self.route}/{name}") + names = frappe.db.get_values(self.doc_type, {"owner": frappe.session.user}, pluck="name") + for name in names: + if self.condition: + doc = frappe.get_doc(self.doc_type, name) + if frappe.safe_eval(self.condition, None, {"doc": doc.as_dict()}): + context.in_view_mode = True + frappe.redirect(f"/{self.route}/{name}") + else: + context.in_view_mode = True + frappe.redirect(f"/{self.route}/{name}") # Show new form when # - User is Guest @@ -633,3 +637,8 @@ def get_link_options(web_form_name, doctype, allow_read_on_all_link_options=Fals raise frappe.PermissionError( _("You don't have permission to access the {0} DocType.").format(doctype) ) + + +@redis_cache(ttl=60 * 60) +def get_published_web_forms() -> dict[str, str]: + return frappe.get_all("Web Form", ["name", "route", "modified"], {"published": 1}) diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index 9a16654085..02e419001c 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -8,6 +8,7 @@ from jinja2.exceptions import TemplateSyntaxError import frappe from frappe import _ from frappe.utils import get_datetime, now, quoted, strip_html +from frappe.utils.caching import redis_cache from frappe.utils.jinja import render_template from frappe.utils.safe_exec import safe_exec from frappe.website.doctype.website_slideshow.website_slideshow import get_slideshow @@ -30,12 +31,6 @@ class WebPage(WebsiteGenerator): if not self.dynamic_route: self.route = quoted(self.route) - def on_update(self): - super().on_update() - - def on_trash(self): - super().on_trash() - def get_context(self, context): context.main_section = get_html_content_based_on_type(self, "main_section", self.content_type) context.source_content_type = self.content_type @@ -247,3 +242,10 @@ def extract_script_and_style_tags(html): style.extract() return str(soup), scripts, styles + + +@redis_cache(ttl=60 * 60) +def get_dynamic_web_pages() -> dict[str, str]: + return frappe.get_all( + "Web Page", fields=["name", "route", "modified"], filters=dict(published=1, dynamic_route=1) + ) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index abfd72ac6f..54ee58ddb9 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,5 +1,6 @@ import frappe from frappe.model.document import get_controller +from frappe.utils.caching import redis_cache from frappe.website.page_renderers.base_template_page import BaseTemplatePage from frappe.website.router import ( get_doctypes_with_web_view, @@ -22,22 +23,9 @@ class DocumentPage(BaseTemplatePage): return False def search_in_doctypes_with_web_view(self): - for doctype in get_doctypes_with_web_view(): - filters = dict(route=self.path) - meta = frappe.get_meta(doctype) - condition_field = self.get_condition_field(meta) - - if condition_field: - filters[condition_field] = 1 - - try: - self.docname = frappe.db.get_value(doctype, filters, "name") - if self.docname: - self.doctype = doctype - return True - except Exception as e: - if not frappe.db.is_missing_column(e): - raise e + if document := _find_matching_document_webview(self.path): + self.doctype, self.docname = document + return True def search_web_page_dynamic_routes(self): d = get_page_info_from_web_page_with_dynamic_routes(self.path) @@ -83,7 +71,8 @@ class DocumentPage(BaseTemplatePage): if prop not in self.context: self.context[prop] = getattr(self.doc, prop, False) - def get_condition_field(self, meta): + @staticmethod + def get_condition_field(meta): condition_field = None if meta.is_published_field: condition_field = meta.is_published_field @@ -92,3 +81,22 @@ class DocumentPage(BaseTemplatePage): condition_field = controller.website.condition_field return condition_field + + +@redis_cache(ttl=60 * 60) +def _find_matching_document_webview(route: str) -> tuple[str, str] | None: + for doctype in get_doctypes_with_web_view(): + filters = dict(route=route) + meta = frappe.get_meta(doctype) + condition_field = DocumentPage.get_condition_field(meta) + + if condition_field: + filters[condition_field] = 1 + + try: + docname = frappe.db.get_value(doctype, filters, "name") + if docname: + return (doctype, docname) + except Exception as e: + if not frappe.db.is_missing_column(e): + raise e diff --git a/frappe/website/router.py b/frappe/website/router.py index 655fcc1357..98be1138e4 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -16,12 +16,11 @@ def get_page_info_from_web_page_with_dynamic_routes(path): """ Query Web Page with dynamic_route = 1 and evaluate if any of the routes match """ + from frappe.website.doctype.web_page.web_page import get_dynamic_web_pages + rules, page_info = [], {} - # build rules from all web page with `dynamic_route = 1` - for d in frappe.get_all( - "Web Page", fields=["name", "route", "modified"], filters=dict(published=1, dynamic_route=1) - ): + for d in get_dynamic_web_pages(): rules.append(Rule("/" + d.route, endpoint=d.name)) d.doctype = "Web Page" page_info[d.name] = d @@ -33,9 +32,10 @@ def get_page_info_from_web_page_with_dynamic_routes(path): def get_page_info_from_web_form(path): """Query published web forms and evaluate if the route matches""" + from frappe.website.doctype.web_form.web_form import get_published_web_forms + rules, page_info = [], {} - web_forms = frappe.get_all("Web Form", ["name", "route", "modified"], {"published": 1}) - for d in web_forms: + for d in get_published_web_forms(): rules.append(Rule(f"/{d.route}", endpoint=d.name)) rules.append(Rule(f"/{d.route}/list", endpoint=d.name)) rules.append(Rule(f"/{d.route}/new", endpoint=d.name)) @@ -315,3 +315,13 @@ def get_doctypes_with_web_view(): def get_start_folders(): return frappe.local.flags.web_pages_folders or ("www", "templates/pages") + + +def clear_routing_cache(): + from frappe.website.doctype.web_form.web_form import get_published_web_forms + from frappe.website.doctype.web_page.web_page import get_dynamic_web_pages + from frappe.website.page_renderers.document_page import _find_matching_document_webview + + _find_matching_document_webview.clear_cache() + get_dynamic_web_pages.clear_cache() + get_published_web_forms.clear_cache() diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 71af463c96..ff8c69639e 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -360,9 +360,13 @@ def get_html_content_based_on_type(doc, fieldname, content_type): def clear_cache(path=None): """Clear website caches :param path: (optional) for the given path""" + from frappe.website.router import clear_routing_cache + for key in ("website_generator_routes", "website_pages", "website_full_index", "sitemap_routes"): frappe.cache().delete_value(key) + clear_routing_cache() + frappe.cache().delete_value("website_404") if path: frappe.cache().hdel("website_redirects", path) diff --git a/frappe/website/web_template/section_with_cards/section_with_cards.json b/frappe/website/web_template/section_with_cards/section_with_cards.json index c891119f97..5501147d89 100644 --- a/frappe/website/web_template/section_with_cards/section_with_cards.json +++ b/frappe/website/web_template/section_with_cards/section_with_cards.json @@ -49,7 +49,7 @@ }, { "fieldname": "card_1_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -79,7 +79,7 @@ }, { "fieldname": "card_2_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -109,7 +109,7 @@ }, { "fieldname": "card_3_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -139,7 +139,7 @@ }, { "fieldname": "card_4_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -169,7 +169,7 @@ }, { "fieldname": "card_5_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -199,7 +199,7 @@ }, { "fieldname": "card_6_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -229,7 +229,7 @@ }, { "fieldname": "card_7_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -259,7 +259,7 @@ }, { "fieldname": "card_8_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -289,13 +289,13 @@ }, { "fieldname": "card_9_url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 } ], "idx": 0, - "modified": "2021-05-03 13:26:34.470232", + "modified": "2023-06-05 13:26:34.470232", "modified_by": "Administrator", "module": "Website", "name": "Section with Cards", diff --git a/frappe/website/web_template/section_with_features/section_with_features.json b/frappe/website/web_template/section_with_features/section_with_features.json index a5734aa293..2683e92aae 100644 --- a/frappe/website/web_template/section_with_features/section_with_features.json +++ b/frappe/website/web_template/section_with_features/section_with_features.json @@ -43,7 +43,7 @@ }, { "fieldname": "url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 }, @@ -55,7 +55,7 @@ } ], "idx": 2, - "modified": "2020-10-26 17:43:08.219285", + "modified": "2023-06-05 13:26:34.470232", "modified_by": "Administrator", "module": "Website", "name": "Section with Features", diff --git a/frappe/website/web_template/section_with_testimonials/section_with_testimonials.json b/frappe/website/web_template/section_with_testimonials/section_with_testimonials.json index c1ba071be2..dd1d3bd0bd 100644 --- a/frappe/website/web_template/section_with_testimonials/section_with_testimonials.json +++ b/frappe/website/web_template/section_with_testimonials/section_with_testimonials.json @@ -56,13 +56,13 @@ }, { "fieldname": "url", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "URL", "reqd": 0 } ], "idx": 0, - "modified": "2022-03-21 15:39:39.044104", + "modified": "2023-06-05 13:26:34.470232", "modified_by": "Administrator", "module": "Website", "name": "Section with Testimonials",