diff --git a/.flake8 b/.flake8 index 4b852abd7c..2de7a154c9 100644 --- a/.flake8 +++ b/.flake8 @@ -1,37 +1,74 @@ [flake8] ignore = + B001, + B007, + B009, + B010, + B950, + E101, + E111, + E114, + E116, + E117, E121, + E122, + E123, + E124, + E125, E126, E127, E128, + E131, + E201, + E202, E203, + E211, + E221, + E222, + E223, + E224, E225, E226, + E228, E231, E241, + E242, E251, E261, + E262, E265, + E266, + E271, + E272, + E273, + E274, + E301, E302, E303, E305, + E306, E402, E501, + E502, + E701, + E702, + E703, E741, + F401, + F403, + F405, + W191, W291, W292, W293, W391, W503, W504, - F403, - B007, - B950, - W191, - E124, # closing bracket, irritating while writing QB code - E131, # continuation line unaligned for hanging indent - E123, # closing bracket does not match indentation of opening bracket's line - E101, # ensured by use of black + E711, + E129, + F841, + E713, + E712, max-line-length = 200 -exclude=.github/helper/semgrep_rules +exclude=,test_*.py diff --git a/.github/helper/flake8.conf b/.github/helper/flake8.conf deleted file mode 100644 index 20d4b912ca..0000000000 --- a/.github/helper/flake8.conf +++ /dev/null @@ -1,75 +0,0 @@ -[flake8] -ignore = - B001, - B007, - B009, - B010, - B950, - E101, - E111, - E114, - E116, - E117, - E121, - E122, - E123, - E124, - E125, - E126, - E127, - E128, - E131, - E201, - E202, - E203, - E211, - E221, - E222, - E223, - E224, - E225, - E226, - E228, - E231, - E241, - E242, - E251, - E261, - E262, - E265, - E266, - E271, - E272, - E273, - E274, - E301, - E302, - E303, - E305, - E306, - E402, - E501, - E502, - E701, - E702, - E703, - E741, - F401, - F403, - F405, - W191, - W291, - W292, - W293, - W391, - W503, - W504, - E711, - E129, - F841, - E713, - E712, - - -max-line-length = 200 -exclude=.github/helper/semgrep_rules,test_*.py diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 6c3ba7db81..01b5407489 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -42,7 +42,7 @@ jobs: - name: 'Setup Environment' uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v3 - name: Validate Docs diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 12bf9eca55..4feaebe15d 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -19,7 +19,7 @@ jobs: node-version: 16 - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up bench and build assets run: | npm install -g yarn diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 9c2b933763..ae101d003b 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -63,7 +63,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Check for valid Python & Merge Conflicts run: | diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 926a87249f..dcc078ad2a 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -66,7 +66,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Check for valid Python & Merge Conflicts run: | diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index f41171784c..1a122a3b12 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -64,7 +64,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Check for valid Python & Merge Conflicts run: | @@ -121,7 +121,6 @@ jobs: DB: mariadb - name: Verify yarn.lock - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | cd ~/frappe-bench/apps/frappe yarn install --immutable --immutable-cache --check-cache diff --git a/.mergify.yml b/.mergify.yml index 85b590ba76..b74648a8f5 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -71,15 +71,6 @@ pull_request_rules: assignees: - "{{ author }}" - - name: backport to develop - conditions: - - label="backport develop" - actions: - backport: - branches: - - develop - assignees: - - "{{ author }}" - name: backport to version-13-pre-release conditions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27fae671c9..0783e94457 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,7 +59,6 @@ repos: hooks: - id: flake8 additional_dependencies: ['flake8-bugbear',] - args: ['--config', '.github/helper/flake8.conf'] ci: autoupdate_schedule: weekly diff --git a/.stylelintrc b/.stylelintrc deleted file mode 100644 index 1e05d1fb41..0000000000 --- a/.stylelintrc +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": ["stylelint-config-recommended"], - "plugins": ["stylelint-scss"], - "rules": { - "at-rule-no-unknown": null, - "scss/at-rule-no-unknown": true, - "no-descending-specificity": null - } -} \ No newline at end of file diff --git a/bandit.yml b/bandit.yml deleted file mode 100644 index b8560e97c8..0000000000 --- a/bandit.yml +++ /dev/null @@ -1 +0,0 @@ -skips: ['E0203', 'B605', 'B404', 'B603', 'B607'] \ No newline at end of file diff --git a/cypress/integration/dashboard.js b/cypress/integration/dashboard.js new file mode 100644 index 0000000000..6eb28567bc --- /dev/null +++ b/cypress/integration/dashboard.js @@ -0,0 +1,50 @@ +describe("Dashboard view", { scrollBehavior: false }, () => { + before(() => { + cy.login(); + cy.visit("/app"); + }); + + it("should load", () => { + const chart = "TODO-YEARLY-TRENDS"; + const dashboard = "TODO-TEST-DASHBOARD"; // check slash in name intentionally. + + cy.insert_doc( + "Dashboard Chart", + { + is_standard: 0, + chart_name: chart, + chart_type: "Count", + document_type: "ToDo", + parent_document_type: "", + based_on: "creation", + group_by_type: "Count", + timespan: "Last Year", + time_interval: "Yearly", + timeseries: 1, + type: "Line", + filters_json: "[]", + }, + true + ); + + cy.insert_doc( + "Dashboard", + { + name: dashboard, + dashboard_name: dashboard, + is_standard: 0, + charts: [ + { + chart: chart, + }, + ], + }, + true + ); + + cy.visit(`/app/dashboard-view/${dashboard}`); + + // expect chart to be loaded + cy.findByText(chart).should("be.visible"); + }); +}); diff --git a/frappe/__init__.py b/frappe/__init__.py index c03b87be1c..84a27642a9 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1018,19 +1018,15 @@ def get_precision( return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency) -def generate_hash(txt: str | None = None, length: int | None = None) -> str: - """Generates random hash for given text + current timestamp + random string.""" - import hashlib - import time +def generate_hash(txt: str | None = None, length: int = 56) -> str: + """Generate random hash using best available randomness source.""" + import math + import secrets - from .utils import random_string + if not length: + length = 56 - digest = hashlib.sha224( - ((txt or "") + repr(time.time()) + repr(random_string(8))).encode() - ).hexdigest() - if length: - digest = digest[:length] - return digest + return secrets.token_hex(math.ceil(length / 2))[:length] def reset_metadata_version(): diff --git a/frappe/app.py b/frappe/app.py index 136b16bff5..0d7fdc1fe1 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -12,13 +12,13 @@ from werkzeug.wrappers import Request, Response import frappe import frappe.api -import frappe.auth import frappe.handler import frappe.monitor import frappe.rate_limiter import frappe.recorder import frappe.utils.response from frappe import _ +from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request from frappe.middlewares import StaticDataMiddleware from frappe.utils import get_site_name, sanitize_html @@ -29,8 +29,6 @@ local_manager = LocalManager(frappe.local) _site = None _sites_path = os.environ.get("SITES_PATH", ".") -SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS") -UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH") @local_manager.middleware @@ -88,7 +86,8 @@ def application(request: Request): log_request(request, response) process_response(response) - frappe.destroy() + if frappe.db: + frappe.db.close() return response @@ -118,7 +117,7 @@ def init_request(request): make_form_dict(request) if request.method != "OPTIONS": - frappe.local.http_request = frappe.auth.HTTPRequest() + frappe.local.http_request = HTTPRequest() def setup_read_only_mode(): diff --git a/frappe/auth.py b/frappe/auth.py index f7ff6f0fe5..e4bde99907 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -20,6 +20,9 @@ from frappe.utils import cint, date_diff, datetime, get_datetime, today from frappe.utils.password import check_password from frappe.website.utils import get_home_page +SAFE_HTTP_METHODS = frozenset(("GET", "HEAD", "OPTIONS")) +UNSAFE_HTTP_METHODS = frozenset(("POST", "PUT", "DELETE", "PATCH")) + class HTTPRequest: def __init__(self): @@ -67,25 +70,21 @@ class HTTPRequest: frappe.local.login_manager = LoginManager() def validate_csrf_token(self): - if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"): - if not frappe.local.session: - return - if ( - not frappe.local.session.data.csrf_token - or frappe.local.session.data.device == "mobile" - or frappe.conf.get("ignore_csrf", None) - ): - # not via boot - return + if ( + not frappe.request + or frappe.request.method not in UNSAFE_HTTP_METHODS + or frappe.conf.ignore_csrf + or not frappe.session + or not (saved_token := frappe.session.data.csrf_token) + or ( + (frappe.get_request_header("X-Frappe-CSRF-Token") or frappe.form_dict.pop("csrf_token", None)) + == saved_token + ) + ): + return - csrf_token = frappe.get_request_header("X-Frappe-CSRF-Token") - if not csrf_token and "csrf_token" in frappe.local.form_dict: - csrf_token = frappe.local.form_dict.csrf_token - del frappe.local.form_dict["csrf_token"] - - if frappe.local.session.data.csrf_token != csrf_token: - frappe.local.flags.disable_traceback = True - frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) + frappe.flags.disable_traceback = True + frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): frappe.local.lang = get_language() @@ -354,10 +353,6 @@ class CookieManager: if not secure and hasattr(frappe.local, "request"): secure = frappe.local.request.scheme == "https" - # Cordova does not work with Lax - if frappe.local.session.data.device == "mobile": - samesite = None - self.cookies[key] = { "value": value, "expires": expires, diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index d4ce92f384..eeddef1865 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -8,11 +8,19 @@ from frappe.desk.notifications import clear_notifications, delete_notification_c common_default_keys = ["__default", "__global"] -doctype_map_keys = ( - "energy_point_rule_map", - "assignment_rule_map", - "milestone_tracker_map", -) +doctypes_for_mapping = { + "Energy Point Rule", + "Assignment Rule", + "Milestone Tracker", + "Document Naming Rule", +} + + +def get_doctype_map_key(doctype): + return frappe.scrub(doctype) + "_map" + + +doctype_map_keys = tuple(map(get_doctype_map_key, doctypes_for_mapping)) bench_cache_keys = ("assets_json",) @@ -66,7 +74,7 @@ doctype_cache_keys = ( "notifications", "workflow", "data_import_column_header_map", -) + doctype_map_keys +) def clear_user_cache(user=None): @@ -161,23 +169,11 @@ def clear_controller_cache(doctype=None): def get_doctype_map(doctype, name, filters=None, order_by=None): - cache = frappe.cache() - cache_key = frappe.scrub(doctype) + "_map" - doctype_map = cache.hget(cache_key, name) - - if doctype_map is not None: - # cached, return - items = json.loads(doctype_map) - else: - # non cached, build cache - try: - items = frappe.get_all(doctype, filters=filters, order_by=order_by) - cache.hset(cache_key, name, json.dumps(items)) - except frappe.db.TableMissingError: - # executed from inside patch, ignore - items = [] - - return items + return frappe.cache().hget( + get_doctype_map_key(doctype), + name, + lambda: frappe.get_all(doctype, filters=filters, order_by=order_by, ignore_ddl=True), + ) def clear_doctype_map(doctype, name): diff --git a/frappe/client.py b/frappe/client.py index f42f73a529..404617b68c 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -333,11 +333,6 @@ def get_js(items): with open(contentpath) as srcfile: code = frappe.utils.cstr(srcfile.read()) - if frappe.local.lang != "en": - messages = frappe.get_lang_dict("jsfile", contentpath) - messages = json.dumps(messages) - code += f"\n\n$.extend(frappe._messages, {messages})" - out.append(code) return out diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index e3bf669630..611592531d 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -432,7 +432,7 @@ class DataExporter: row[_column_start_end.start + i + 1] = value def build_response_as_excel(self): - filename = frappe.generate_hash("", 10) + filename = frappe.generate_hash(length=10) with open(filename, "wb") as f: f.write(cstr(self.writer.getvalue()).encode("utf-8")) f = open(filename) diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index af8c711ab5..978f5792dd 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -97,7 +97,7 @@ class TestImporter(FrappeTestCase): def test_data_import_update(self): existing_doc = frappe.get_doc( doctype=doctype_name, - title=frappe.generate_hash(doctype_name, 8), + title=frappe.generate_hash(length=8), table_field_1=[{"child_title": "child title to update"}], ) existing_doc.save() diff --git a/frappe/core/doctype/doctype/boilerplate/controller.js b/frappe/core/doctype/doctype/boilerplate/controller.js index 6d9fb2a514..0e3dcd2e26 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller.js +++ b/frappe/core/doctype/doctype/boilerplate/controller.js @@ -1,8 +1,8 @@ // Copyright (c) {year}, {app_publisher} and contributors // For license information, please see license.txt -frappe.ui.form.on('{doctype}', {{ - // refresh: function(frm) {{ +// frappe.ui.form.on("{doctype}", {{ +// refresh(frm) {{ - // }} -}}); +// }}, +// }}); diff --git a/frappe/core/doctype/doctype/boilerplate/controller_list.js b/frappe/core/doctype/doctype/boilerplate/controller_list.js index b1f6d12008..3740cfa85d 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller_list.js +++ b/frappe/core/doctype/doctype/boilerplate/controller_list.js @@ -1,5 +1,5 @@ /* eslint-disable */ -frappe.listview_settings['{doctype}'] = {{ - // add_fields: ["status"], - // filters:[["status","=", "Open"]] -}}; +// frappe.listview_settings["{doctype}"] = {{ +// add_fields: ["status"], +// filters: [["status","=", "Open"]], +// }}; diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 984e78ae5c..3c5d5ffdcb 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -24,6 +24,7 @@ "custom", "beta", "is_virtual", + "queue_in_background", "fields_section_break", "fields", "sb1", @@ -600,6 +601,13 @@ "fieldtype": "Check", "label": "Make Attachments Public by Default" }, + { + "default": "0", + "depends_on": "eval: doc.is_submittable", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" + }, { "fieldname": "default_view", "fieldtype": "Select", diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index acc5c4871d..f4760ec3c6 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -329,7 +329,7 @@ class DocType(Document): "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name) ) for p in parent_list: - frappe.db.update("DocType", p.parent, {}, for_update=False) + frappe.db.set_value("DocType", p.parent, {}, for_update=False) def scrub_field_names(self): """Sluggify fieldnames if not set from Label.""" diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 3722e5d1fa..2e74fd3a6a 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -670,6 +670,9 @@ class TestDocType(FrappeTestCase): self.assertEqual(test_json.test_json_field["hello"], "world") + def test_no_delete_doc(self): + self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address") + @patch.dict(frappe.conf, {"developer_mode": 1}) def test_custom_field_deletion(self): """Custom child tables whose doctype doesn't exist should be auto deleted.""" diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 3fecf26ade..598de98dbb 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -12,6 +12,15 @@ class DocumentNamingRule(Document): def validate(self): self.validate_fields_in_conditions() + def clear_doctype_map(self): + frappe.cache_manager.clear_doctype_map(self.doctype, self.document_type) + + def on_update(self): + self.clear_doctype_map() + + def on_trash(self): + self.clear_doctype_map() + def validate_fields_in_conditions(self): if self.has_value_changed("document_type"): docfields = [x.fieldname for x in frappe.get_meta(self.document_type).fields] diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 4a519dcaf4..f0f2cdaae8 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -17,6 +17,7 @@ DEFAULT_LOGTYPES_RETENTION = { "Error Snapshot": 30, "Scheduled Job Log": 90, "Route History": 90, + "Submission Queue": 30, } @@ -68,6 +69,9 @@ class LogSettings(Document): added_logtypes = set() for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items(): if logtype not in existing_logtypes and _supports_log_clearing(logtype): + if not frappe.db.exists("DocType", logtype): + continue + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)}) added_logtypes.add(logtype) @@ -151,6 +155,7 @@ LOG_DOCTYPES = [ "Email Queue Recipient", "Error Snapshot", "Error Log", + "Submission Queue", ] diff --git a/frappe/core/doctype/rq_job/rq_job.py b/frappe/core/doctype/rq_job/rq_job.py index 7e1c35a0e6..f05611fe7d 100644 --- a/frappe/core/doctype/rq_job/rq_job.py +++ b/frappe/core/doctype/rq_job/rq_job.py @@ -5,10 +5,12 @@ import functools import re from rq.command import send_stop_job_command +from rq.exceptions import InvalidJobOperation from rq.job import Job from rq.queue import Queue import frappe +from frappe import _ from frappe.model.document import Document from frappe.utils import ( cint, @@ -93,7 +95,10 @@ class RQJob(Document): @check_permissions def stop_job(self): - send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id) + try: + send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id) + except InvalidJobOperation: + frappe.msgprint(_("Job is not running."), title=_("Invalid Operation")) @staticmethod def get_count(args) -> int: diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index ae0691fa61..460aa08941 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -19,12 +19,11 @@ class TestRQJob(FrappeTestCase): @timeout(seconds=20) def check_status(self, job: Job, status, wait=True): - if wait: - while True: - if job.is_queued or job.is_started: - time.sleep(0.2) - else: - break + while wait: + if not (job.is_queued or job.is_started): + break + time.sleep(0.2) + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) def test_serialization(self): @@ -69,7 +68,7 @@ class TestRQJob(FrappeTestCase): self.assertGreaterEqual(len(non_failed_jobs), 1) # Create a slow job and check if it's stuck in "Started" - job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=1000) + job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=10) time.sleep(3) self.check_status(job, "started", wait=False) stop_job(job_id=job.id) @@ -84,8 +83,8 @@ class TestRQJob(FrappeTestCase): def test_is_enqueued(self): + dummy_job = frappe.enqueue(self.BG_JOB, sleep=10, queue="short") job_name = "uniq_test_job" - dummy_job = frappe.enqueue(self.BG_JOB, sleep=100, queue="short") actual_job = frappe.enqueue(self.BG_JOB, job_name=job_name, queue="short") self.assertTrue(is_job_queued(job_name)) diff --git a/frappe/core/doctype/rq_worker/rq_worker.json b/frappe/core/doctype/rq_worker/rq_worker.json index ea65abd482..d9a5a23f67 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.json +++ b/frappe/core/doctype/rq_worker/rq_worker.json @@ -20,7 +20,8 @@ "column_break_12", "birth_date", "last_heartbeat", - "total_working_time" + "total_working_time", + "utilization_percent" ], "fields": [ { @@ -59,7 +60,6 @@ { "fieldname": "successful_job_count", "fieldtype": "Int", - "in_list_view": 1, "label": "Successful Job Count" }, { @@ -102,12 +102,18 @@ { "fieldname": "column_break_12", "fieldtype": "Column Break" + }, + { + "fieldname": "utilization_percent", + "fieldtype": "Percent", + "in_list_view": 1, + "label": "Utilization %" } ], "in_create": 1, "is_virtual": 1, "links": [], - "modified": "2022-09-11 05:02:53.981705", + "modified": "2022-11-14 15:35:32.786012", "modified_by": "Administrator", "module": "Core", "name": "RQ Worker", diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py index b2d1f1209d..3de0c8f7fc 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.py +++ b/frappe/core/doctype/rq_worker/rq_worker.py @@ -1,6 +1,9 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +import datetime +from contextlib import suppress + from rq import Worker import frappe @@ -66,4 +69,11 @@ def serialize_worker(worker: Worker) -> frappe._dict: _comment_count=0, modified=convert_utc_to_user_timezone(worker.last_heartbeat), creation=convert_utc_to_user_timezone(worker.birth_date), + utilization_percent=compute_utilization(worker), ) + + +def compute_utilization(worker: Worker) -> float: + with suppress(Exception): + total_time = (datetime.datetime.utcnow() - worker.birth_date).total_seconds() + return worker.total_working_time / total_time * 100 diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 7002de9691..3abc53bd52 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -211,3 +211,25 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run() """ script.save() script.execute_method() + + def test_scripts_all_the_way_down(self): + # why not + script = frappe.get_doc( + doctype="Server Script", + name="test_nested_scripts_1", + script_type="API", + api_method="test_nested_scripts_1", + script=f"""log("nothing")""", + ) + script.insert() + script.execute_method() + + script = frappe.get_doc( + doctype="Server Script", + name="test_nested_scripts_2", + script_type="API", + api_method="test_nested_scripts_2", + script=f"""frappe.call("test_nested_scripts_1")""", + ) + script.insert() + script.execute_method() diff --git a/frappe/core/doctype/submission_queue/__init__.py b/frappe/core/doctype/submission_queue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js new file mode 100644 index 0000000000..93d6b981dc --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -0,0 +1,14 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Submission Queue", { + refresh: function (frm) { + if (frm.doc.status === "Queued" && frm.doc.job_id) { + frm.add_custom_button(__("Unlock Reference Document"), () => { + frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => { + frm.call("unlock_doc"); + }); + }); + } + }, +}); diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json new file mode 100644 index 0000000000..d1f66ffa13 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -0,0 +1,123 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2022-10-04 00:41:00.028163", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "status", + "created_at", + "enqueued_by", + "job_id", + "column_break_5", + "ended_at", + "ref_doctype", + "ref_docname", + "section_break_8", + "exception" + ], + "fields": [ + { + "fieldname": "job_id", + "fieldtype": "Data", + "label": "Job Id", + "read_only": 1 + }, + { + "fieldname": "ref_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Reference DocType", + "options": "DocType", + "read_only": 1 + }, + { + "fieldname": "ref_docname", + "fieldtype": "Dynamic Link", + "in_list_view": 1, + "label": "Reference Docname", + "options": "ref_doctype", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "in_list_view": 1, + "label": "Status", + "options": "Queued\nFinished\nFailed", + "read_only": 1 + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_8", + "fieldtype": "Section Break" + }, + { + "fieldname": "enqueued_by", + "fieldtype": "Data", + "is_virtual": 1, + "label": "Enqueued By", + "read_only": 1 + }, + { + "fieldname": "ended_at", + "fieldtype": "Datetime", + "label": "Ended At", + "read_only": 1 + }, + { + "fieldname": "created_at", + "fieldtype": "Datetime", + "is_virtual": 1, + "label": "Created At", + "read_only": 1 + }, + { + "fieldname": "exception", + "fieldtype": "Text", + "label": "Exception", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2022-11-12 16:48:37.797232", + "modified_by": "Administrator", + "module": "Core", + "name": "Submission Queue", + "naming_rule": "Random", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [ + { + "color": "Blue", + "title": "Queued" + }, + { + "color": "Red", + "title": "Failed" + }, + { + "color": "Green", + "title": "Finished" + } + ] +} \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py new file mode 100644 index 0000000000..2bb4200a87 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -0,0 +1,193 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +from urllib.parse import quote + +from rq import get_current_job +from rq.exceptions import NoSuchJobError +from rq.job import Job + +import frappe +from frappe import _ +from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification +from frappe.model.document import Document +from frappe.monitor import add_data_to_monitor +from frappe.utils import now, time_diff_in_seconds +from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.data import cint + + +class SubmissionQueue(Document): + @property + def created_at(self): + return self.creation + + @property + def enqueued_by(self): + return self.owner + + @property + def queued_doc(self): + return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Submission Queue") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + + def insert(self, to_be_queued_doc: Document, action: str): + self.to_be_queued_doc = to_be_queued_doc + self.action_for_queuing = action + super().insert(ignore_permissions=True) + + def lock(self): + self.queued_doc.lock() + + def unlock(self): + self.queued_doc.unlock() + + def update_job_id(self, job_id): + frappe.db.set_value( + self.doctype, + self.name, + {"job_id": job_id}, + update_modified=False, + ) + frappe.db.commit() + + def after_insert(self): + self.queue_action( + "background_submission", + to_be_queued_doc=self.queued_doc, + action_for_queuing=self.action_for_queuing, + timeout=600, + enqueue_after_commit=True, + ) + + def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): + # Set the job id for that submission doctype + self.update_job_id(get_current_job().id) + _action = action_for_queuing.lower() + if _action == "update": + _action = "submit" + + try: + getattr(to_be_queued_doc, _action)() + add_data_to_monitor( + doctype=to_be_queued_doc.doctype, + docname=to_be_queued_doc.name, + action=_action, + execution_time=time_diff_in_seconds(now(), self.created_at), + enqueued_by=self.enqueued_by, + ) + values = {"status": "Finished"} + except Exception: + values = {"status": "Failed", "exception": frappe.get_traceback()} + frappe.db.rollback() + + values["ended_at"] = now() + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + self.notify(values["status"], action_for_queuing) + + def notify(self, submission_status: str, action: str): + if submission_status == "Failed": + doctype = self.doctype + docname = self.name + message = _("Submission of {0} {1} with action {2} failed") + else: + doctype = self.ref_doctype + docname = self.ref_docname + message = _("Submission of {0} {1} with action {2} completed successfully") + + message = message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ) + time_diff = time_diff_in_seconds(now(), self.created_at) + if cint(time_diff) <= 60: + frappe.publish_realtime( + "msgprint", + { + "message": message + + f". View it here", + "alert": True, + "indicator": "red" if submission_status == "Failed" else "green", + }, + user=self.enqueued_by, + ) + else: + notification_doc = { + "type": "Alert", + "document_type": doctype, + "document_name": docname, + "subject": message, + } + + notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") + enqueue_create_notification([notify_to], notification_doc) + + def _unlock_reference_doc(self): + """ + Only execute if self.job_id is defined. + """ + try: + job = Job.fetch(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + exc = job.exc_info + except NoSuchJobError: + exc = None + status = "failed" + + if status in ("queued", "started"): + frappe.msgprint(_("Document in queue for execution!")) + return + + self.queued_doc.unlock() + values = ( + {"status": "Finished"} if status == "finished" else {"status": "Failed", "exception": exc} + ) + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + frappe.msgprint(_("Document Unlocked")) + + @frappe.whitelist() + def unlock_doc(self): + # NOTE: this can lead to some weird unlocking/locking behaviours. + # for example: hitting unlock on a submission could lead to unlocking of another submission + # of the same reference document. + + if self.status != "Queued" and not self.job_id: + return + + self._unlock_reference_doc() + + +def queue_submission(doc: Document, action: str, alert: bool = True): + queue = frappe.new_doc("Submission Queue") + queue.state = "Queued" + queue.ref_doctype = doc.doctype + queue.ref_docname = doc.name + queue.insert(doc, action) + + if alert: + frappe.msgprint( + _("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) + + +@frappe.whitelist() +def get_latest_submissions(doctype, docname): + # NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere + # hence assuming modified will be equal to creation for submission queue documents + + dt = "Submission Queue" + filters = {"ref_doctype": doctype, "ref_docname": docname} + return { + "latest_submission": frappe.db.get_value(dt, filters), + "latest_failed_submission": frappe.db.get_value(dt, filters | {"status": "Failed"}), + } diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py new file mode 100644 index 0000000000..c057bd22e1 --- /dev/null +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -0,0 +1,51 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +import time +import typing + +import frappe +from frappe.tests.utils import FrappeTestCase, timeout +from frappe.utils.background_jobs import get_queue + +if typing.TYPE_CHECKING: + from rq.job import Job + + +class TestSubmissionQueue(FrappeTestCase): + queue = get_queue(qtype="default") + + @timeout(seconds=20) + def check_status(self, job: "Job", status, wait=True): + if wait: + while True: + if job.is_queued or job.is_started: + time.sleep(0.2) + else: + break + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) + + def test_queue_operation(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype + from frappe.core.doctype.submission_queue.submission_queue import queue_submission + + if not frappe.db.table_exists("Test Submission Queue", cached=False): + doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) + doc.insert() + + d = frappe.new_doc("Test Submission Queue") + d.update({"some_fieldname": "Random"}) + d.insert() + + frappe.db.commit() + queue_submission(d, "submit") + frappe.db.commit() + + # Waiting for execution + time.sleep(4) + submission_queue = frappe.get_last_doc("Submission Queue") + + # Test queueing / starting + job = self.queue.fetch_job(submission_queue.job_id) + # Test completion + self.check_status(job, status="finished") diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 092f7fa45d..3dc43ccc33 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -471,7 +471,7 @@ class User(Document): frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False) # set email - frappe.db.update("User", new_name, "email", new_name) + frappe.db.set_value("User", new_name, "email", new_name) def append_roles(self, *roles): """Add roles to user""" diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 46c9e0aca2..45c1e44fa1 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -19,7 +19,6 @@ from frappe.permissions import ( setup_custom_perms, update_permission_property, ) -from frappe.translate import send_translations from frappe.utils.user import get_users_with_role as _get_user_with_role not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"] @@ -28,7 +27,6 @@ not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Tran @frappe.whitelist() def get_roles_and_doctypes(): frappe.only_for("System Manager") - send_translations(frappe.get_lang_dict("doctype", "DocPerm")) active_domains = frappe.get_active_domains() diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 759a9e1b3a..6f94f32256 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -149,6 +149,10 @@ frappe.ui.form.on("Customize Form", { const is_autoname_autoincrement = frm.doc.autoname === "autoincrement"; frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); + frm.toggle_display( + ["queue_in_background"], + frappe.get_meta(frm.doc.doc_type).is_submittable || 0 + ); } frm.events.setup_export(frm); diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 4840184966..b9fb52d1dc 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -20,6 +20,7 @@ "track_views", "allow_auto_repeat", "allow_import", + "queue_in_background", "fields_section_break", "fields", "naming_section", @@ -341,6 +342,12 @@ "fieldtype": "Check", "label": "Make Attachments Public by Default" }, + { + "default": "0", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" + }, { "fieldname": "default_view", "fieldtype": "Select", @@ -367,7 +374,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-08-30 11:45:16.772277", + "modified": "2022-10-30 23:39:49.628093", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index be27ebbc0b..bdd18cddfa 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -571,6 +571,7 @@ doctype_properties = { "allow_copy": "Check", "istable": "Check", "quick_entry": "Check", + "queue_in_background": "Check", "editable_grid": "Check", "max_attachments": "Int", "make_attachments_public": "Check", diff --git a/frappe/database/database.py b/frappe/database/database.py index 8e6a066db1..dfcc9dfe58 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -7,7 +7,7 @@ import random import re import string import traceback -from contextlib import contextmanager +from contextlib import contextmanager, suppress from time import time from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder @@ -29,7 +29,8 @@ from frappe.exceptions import DoesNotExistError, ImplicitCommitError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.utils import cast as cast_fieldtype -from frappe.utils import get_datetime, get_table_name, getdate, now, sbool +from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool +from frappe.utils.deprecations import deprecated IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") @@ -114,6 +115,17 @@ class Database: self._cursor = self._conn.cursor() frappe.local.rollback_observers = [] + try: + if execution_timeout := get_query_execution_timeout(): + self.set_execution_timeout(execution_timeout) + except Exception as e: + frappe.logger("database").warning(f"Couldn't set execution timeout {e}") + + def set_execution_timeout(self, seconds: int): + """Set session speicifc timeout on exeuction of statements. + If any statement takes more time it will be killed along with entire transaction.""" + raise NotImplementedError + def use(self, db_name): """`USE` db_name.""" self._conn.select_db(db_name) @@ -135,12 +147,11 @@ class Database: self, query: Query, values: QueryValues = EmptyQueryValues, + *, as_dict=0, as_list=0, - formatted=0, debug=0, ignore_ddl=0, - as_utf8=0, auto_commit=0, update=None, explain=False, @@ -153,10 +164,8 @@ class Database: :param values: Tuple / List / Dict of values to be escaped and substituted in the query. :param as_dict: Return as a dictionary. :param as_list: Always return as a list. - :param formatted: Format values like date etc. :param debug: Print query and `EXPLAIN` in debug log. :param ignore_ddl: Catch exception if table, column missing. - :param as_utf8: Encode values as UTF 8. :param auto_commit: Commit after executing the query. :param update: Update this dict to all rows (if returned `as_dict`). :param run: Returns query without executing it if False. @@ -264,13 +273,13 @@ class Database: # scrub output if required if as_dict: - ret = self.fetch_as_dict(formatted, as_utf8) + ret = self.fetch_as_dict() if update: for r in ret: r.update(update) return ret - elif as_list or as_utf8: - return self.convert_to_lists(self.last_result, formatted, as_utf8) + elif as_list: + return self.convert_to_lists(self.last_result) return self.last_result def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: @@ -377,56 +386,27 @@ class Database: ): raise ImplicitCommitError("This statement can cause implicit commit") - def fetch_as_dict(self, formatted=0, as_utf8=0) -> list[frappe._dict]: + def fetch_as_dict(self) -> list[frappe._dict]: """Internal. Converts results to dict.""" result = self.last_result - ret = [] if result: keys = [column[0] for column in self._cursor.description] - for r in result: - values = [] - for value in r: - if as_utf8 and isinstance(value, str): - value = value.encode("utf-8") - values.append(value) - - ret.append(frappe._dict(zip(keys, values))) - return ret + return [frappe._dict(zip(keys, row)) for row in result] @staticmethod def clear_db_table_cache(query): if query and is_query_type(query, ("drop", "create")): frappe.cache().delete_key("db_tables") - @staticmethod - def needs_formatting(result, formatted): - """Returns true if the first row in the result has a Date, Datetime, Long Int.""" - if result and result[0]: - for v in result[0]: - if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)): - return True - if formatted and isinstance(v, (int, float)): - return True - - return False - def get_description(self): """Returns result metadata.""" return self._cursor.description @staticmethod - def convert_to_lists(res, formatted=0, as_utf8=0): + def convert_to_lists(res): """Convert tuple output to lists (internal).""" - nres = [] - for r in res: - nr = [] - for val in r: - if as_utf8 and isinstance(val, str): - val = val.encode("utf-8") - nr.append(val) - nres.append(nr) - return nres + return [[value for value in row] for row in res] def get(self, doctype, filters=None, as_dict=True, cache=False): """Returns `get_value` with fieldname='*'""" @@ -786,13 +766,11 @@ class Database: distinct=False, limit=None, ): - field_objects = [] query = frappe.qb.engine.get_query( table=doctype, filters=filters, orderby=order_by, for_update=for_update, - field_objects=field_objects, fields=fields, distinct=distinct, limit=limit, @@ -828,10 +806,6 @@ class Database: ).run(debug=debug, run=run, as_dict=as_dict) return {} - def update(self, *args, **kwargs): - """Update multiple values. Alias for `set_value`.""" - return self.set_value(*args, **kwargs) - def set_value( self, dt, @@ -857,7 +831,6 @@ class Database: :param modified_by: Set this user as `modified_by`. :param update_modified: default True. Set as false, if you don't want to update the timestamp. :param debug: Print the query in the developer / js console. - :param for_update: [DEPRECATED] This function now performs updates in single query, locking is not required. """ is_single_doctype = not (dn and dt != dn) to_update = field if isinstance(field, dict) else {field: val} @@ -897,30 +870,6 @@ class Database: if dt in self.value_cache: del self.value_cache[dt] - @staticmethod - def set(doc, field, val): - """Set value in document. **Avoid**""" - doc.db_set(field, val) - - def touch(self, doctype, docname): - """Update the modified timestamp of this document.""" - modified = now() - DocType = frappe.qb.DocType(doctype) - frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run() - return modified - - @staticmethod - def set_temp(value): - """Set a temperory value and return a key.""" - key = frappe.generate_hash() - frappe.cache().hset("temp", key, value) - return key - - @staticmethod - def get_temp(key): - """Return the temperory value and delete it.""" - return frappe.cache().hget("temp", key) - def set_global(self, key, val, user="__global"): """Save a global key value. Global values will be automatically set if they match fieldname.""" self.set_default(key, val, user) @@ -1080,7 +1029,7 @@ class Database: return getdate(date).strftime("%Y-%m-%d") @staticmethod - def format_datetime(datetime): + def format_datetime(datetime): # noqa: F811 if not datetime: return FallBackDateTimeStr @@ -1224,9 +1173,6 @@ class Database: """ return self.sql_ddl(f"truncate `{get_table_name(doctype)}`") - def clear_table(self, doctype): - return self.truncate(doctype) - def get_last_created(self, doctype): last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc") if last_record: @@ -1342,3 +1288,28 @@ def savepoint(catch: type | tuple[type, ...] = Exception): frappe.db.rollback(save_point=savepoint) else: frappe.db.release_savepoint(savepoint) + + +def get_query_execution_timeout() -> int: + """Get execution timeout based on current timeout in different contexts. + + HTTP requests: HTTP timeout or a default (300) + Background jobs: Job timeout + Console/Commands: No timeout = 0. + + Note: Timeout adds 1.5x as "safety factor" + """ + from rq import get_current_job + + if not frappe.conf.get("enable_db_statement_timeout"): + return 0 + + # Zero means no timeout, which is the default value in db. + timeout = 0 + with suppress(Exception): + if getattr(frappe.local, "request", None): + timeout = frappe.conf.http_timeout or 300 + elif job := get_current_job(): + timeout = job.timeout + + return int(cint(timeout) * 1.5) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 1df9877eb1..322c355357 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -68,6 +68,10 @@ class MariaDBExceptionUtil: def is_syntax_error(e: pymysql.Error) -> bool: return e.args[0] == ER.PARSE_ERROR + @staticmethod + def is_statement_timeout(e: pymysql.Error) -> bool: + return e.args[0] == 1969 + @staticmethod def is_data_too_long(e: pymysql.Error) -> bool: return e.args[0] == ER.DATA_TOO_LONG @@ -102,6 +106,9 @@ class MariaDBConnectionUtil: def create_connection(self): return pymysql.connect(**self.get_connection_settings()) + def set_execution_timeout(self, seconds: int): + self.sql("set session max_statement_time = %s", int(seconds)) + def get_connection_settings(self) -> dict: conn_settings = { "host": self.host, diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 3b3612c0e4..d082afceaf 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -99,6 +99,10 @@ class PostgresExceptionUtil: def is_duplicate_fieldname(e): return getattr(e, "pgcode", None) == DUPLICATE_COLUMN + @staticmethod + def is_statement_timeout(e): + return PostgresDatabase.is_timedout(e) or isinstance(e, frappe.QueryTimeoutError) + @staticmethod def is_data_too_long(e): return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION @@ -161,6 +165,10 @@ class PostgresDatabase(PostgresExceptionUtil, Database): return conn + def set_execution_timeout(self, seconds: int): + # Postgres expects milliseconds as input + self.sql("set local statement_timeout = %s", int(seconds) * 1000) + def escape(self, s, percent=True): """Escape quotes and percent in given string.""" if isinstance(s, bytes): diff --git a/frappe/database/query.py b/frappe/database/query.py index 8d1693c199..a9dab02744 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -1,3 +1,4 @@ +import itertools import operator import re from ast import literal_eval @@ -10,11 +11,11 @@ from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder import frappe from frappe import _ -from frappe.database.utils import is_pypika_function_object +from frappe.database.utils import NestedSetHierarchy, is_pypika_function_object from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table, functions from frappe.query_builder.functions import Function, SqlFunctions -from frappe.query_builder.utils import PseudoColumn +from frappe.query_builder.utils import PseudoColumnMapper from frappe.utils.data import MARIADB_SPECIFIC_COMMENT if TYPE_CHECKING: @@ -25,7 +26,6 @@ WORDS_PATTERN = re.compile(r"\w+") BRACKETS_PATTERN = re.compile(r"\(.*?\)|$") SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions] COMMA_PATTERN = re.compile(r",\s*(?![^()]*\))") -TABLE_PATTERN = re.compile(r"`\btab\w+") def like(key: Field, value: str) -> frappe.qb: @@ -164,8 +164,42 @@ def has_function(field): def table_from_string(table: str) -> "DocType": - table_name = table.split("`", maxsplit=1)[1].split(".")[0][3:] - return frappe.qb.DocType(table_name=table_name.replace("`", "")) + if frappe.db.db_type == "postgres": + table_name = table.split('"', maxsplit=1)[1].split(".")[0][3:].replace('"', "") + else: + table_name = table.split("`", maxsplit=1)[1].split(".")[0][3:].replace("`", "") + return frappe.qb.DocType(table_name=table_name) + + +def get_nested_set_hierarchy_result(hierarchy: str, field: str, table: str): + ref_doctype = table + try: + lft, rgt = ( + frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == field).run()[0] + ) + except IndexError: + lft, rgt = None, None + + if hierarchy in ("descendants of", "not descendants of"): + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") > lft) + .where(Field("rgt") < rgt) + .orderby(Field("lft"), order=Order.asc) + .run() + ) + else: + # Get ancestor elements of a DocType with a tree structure + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") < lft) + .where(Field("rgt") > rgt) + .orderby(Field("lft"), order=Order.desc) + .run() + ) + return result # default operators @@ -190,7 +224,7 @@ OPERATOR_MAP: dict[str, Callable] = { "between": func_between, "is": func_is, "timespan": func_timespan, - # TODO: Add support for nested set + "nested_set": NestedSetHierarchy, # TODO: Add support for custom operators (WIP) - via filters_config hooks } @@ -348,7 +382,24 @@ class Engine: if not isinstance(key, str): conditions = conditions.where(self.make_function_for_filters(key, value)) continue + # Nested set support if isinstance(value, (list, tuple)): + if value[0] in self.OPERATOR_MAP["nested_set"]: + hierarchy, _field = value + result = get_nested_set_hierarchy_result(hierarchy, _field, table) + _operator = ( + self.OPERATOR_MAP["not in"] + if hierarchy in ("not ancestors of", "not descendants of") + else self.OPERATOR_MAP["in"] + ) + if result: + result = list(itertools.chain.from_iterable(result)) + conditions = conditions.where(_operator(getattr(table, key), result)) + else: + conditions = conditions.where(_operator(getattr(table, key), ("",))) + # Allow additional conditions + break + _operator = self.OPERATOR_MAP[value[0].casefold()] _value = value[1] if value[1] else ("",) conditions = conditions.where(_operator(getattr(table, key), _value)) @@ -416,11 +467,16 @@ class Engine: if isinstance(operator_mapping, BuiltinFunctionType): has_primitive_operator = True field = operator_mapping( - *map(lambda field: Field(field.strip()), arg.split(_operator)), + *map( + lambda field: Field(field.strip()) + if "`" not in field + else PseudoColumnMapper(field.strip()), + arg.split(_operator), + ), ) field = ( - (Field(initial_fields) if "`" not in initial_fields else PseudoColumn(initial_fields)) + (Field(initial_fields) if "`" not in initial_fields else PseudoColumnMapper(initial_fields)) if not has_primitive_operator else field ) @@ -500,18 +556,14 @@ class Engine: alias = None if " as " in field: field, alias = field.split(" as ") - self.fieldname, linked_fieldname = field.split(".") - linked_field = frappe.get_meta(doctype, cached=True).get_field(self.fieldname) - try: - self.linked_doctype = linked_field.options - except AttributeError: - return fields - field = f"`tab{self.linked_doctype}`.`{linked_fieldname}`" - if alias: - field = f"{field} as {alias}" - _fields.append(field) + fieldname, linked_fieldname = field.split(".") + linked_doctype = frappe.get_meta(doctype).get_field(fieldname).options - return _fields + field = f"`tab{linked_doctype}`.`{linked_fieldname}`" + if alias: + field = f"{field} {alias}" + _fields.append(field) + return _fields def sanitize_fields(self, fields: str | list | tuple): is_mariadb = frappe.db.db_type == "mariadb" @@ -531,21 +583,21 @@ class Engine: return fields - def get_list_fields(self, fields: list) -> list: + def get_list_fields(self, table: str, fields: list) -> list: updated_fields = [] if issubclass(type(fields), Criterion) or "*" in fields: return fields - # fields = self.get_fieldnames_from_child_table(doctype=table, fields=fields) + fields = self.get_fieldnames_from_child_table(doctype=table, fields=fields) for field in fields: if not isinstance(field, Criterion) and field: if " as " in field: field, reference = field.split(" as ") if "`" in field: - updated_fields.append(PseudoColumn(f"{field} as {reference}")) + updated_fields.append(PseudoColumnMapper(f"{field} {reference}")) else: updated_fields.append(Field(field.strip()).as_(reference)) elif "`" in str(field): - updated_fields.append(PseudoColumn(field.strip())) + updated_fields.append(PseudoColumnMapper(field.strip())) else: updated_fields.append(Field(field)) return updated_fields @@ -554,16 +606,16 @@ class Engine: if fields == "*": return fields if "`" in fields: - fields = PseudoColumn(fields) + fields = PseudoColumnMapper(fields) if " as " in str(fields): fields, reference = str(fields).split(" as ") if "`" in str(fields): - fields = PseudoColumn(f"{fields} as {reference}") + fields = PseudoColumnMapper(f"{fields} {reference}") else: fields = Field(fields).as_(reference) return fields - def set_fields(self, fields, **kwargs) -> list: + def set_fields(self, table: str, fields, **kwargs) -> list: fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name" fields = self.sanitize_fields(fields) if isinstance(fields, list) and None in fields and Field not in fields: @@ -591,7 +643,7 @@ class Engine: if is_str: fields = self.get_string_fields(fields) if not is_str and fields: - fields = self.get_list_fields(fields) + fields = self.get_list_fields(table, fields) # Need to check instance again since fields modified. if not isinstance(fields, (list, tuple, set)): @@ -600,51 +652,72 @@ class Engine: fields.extend(function_objects) return fields - def join_(self, criterion, fields, table, join): + def join_child_tables( + self, + criterion: Criterion, + join_type: str, + child_table: Table, + parent_table: Table, + ) -> Criterion: + if self.joined_tables.get(join_type) != child_table: + criterion = getattr(criterion, join_type)(child_table).on( + (child_table.parent == parent_table.name) + & (child_table.parenttype == TAB_PATTERN.sub("", parent_table._table_name)) + ) + self.joined_tables[join_type] = child_table + return criterion + + def join(self, criterion, fields, table, join_type): """Handles all join operations on criterion objects""" has_join = False - joined_tables = {} + table_pattern = ( + re.compile(r"`\btab\w+") if frappe.db.db_type == "mariadb" else re.compile(r'"\btab\w+') + ) + + def _update_pypika_fields(field): + if not is_pypika_function_object(field): + field = field if isinstance(field, (str, PseudoColumnMapper)) else field.get_sql() + if not table_pattern.search(str(field)): + if isinstance(field, PseudoColumnMapper): + field = field.get_sql() + return getattr(frappe.qb.DocType(table), field) + else: + return field + else: + field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args] + return field if not isinstance(fields, Criterion): for field in fields: # Only perform this bit if foreign doctype in fields if ( not is_pypika_function_object(field) - and str(field).startswith("`tab") - and (f"`tab{table}`" not in str(field)) + and (str(field).startswith('"tab') or str(field).startswith("`tab")) + and (f"`tab{table}`" not in str(field) and f'tab{table}"' not in str(field)) ): has_join = True - table_to_join_on = table_from_string(str(field)) - if joined_tables.get(join) != table_to_join_on: - criterion = getattr(criterion, join)(table_to_join_on).on( - getattr(table_to_join_on, "parent") == getattr(frappe.qb.DocType(table), "name") - ) - joined_tables[join] = table_to_join_on + child_table = table_from_string(str(field)) + parent_table = frappe.qb.DocType(table) if not isinstance(table, Table) else table + criterion = self.join_child_tables( + criterion=criterion, + join_type=join_type, + child_table=child_table, + parent_table=parent_table, + ) if has_join: - - def _update_pypika_fields(field): - if not is_pypika_function_object(field): - field = field if isinstance(field, (str, PseudoColumn)) else field.get_sql() - if not TABLE_PATTERN.search(str(field)): - if isinstance(field, PseudoColumn): - field = field.get_sql() - return getattr(frappe.qb.DocType(table), field) - else: - return field - else: - field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args] - return field - fields = [_update_pypika_fields(field) for field in fields] if len(self.tables) > 1: - primary_table = self.tables.pop(table) - for table_object in self.tables.values(): - if joined_tables.get("left_join") != table_object: - criterion = getattr(criterion, join)(table_object).on( - table_object.parent == primary_table.name - ) + parent_table = self.tables[table] + child_tables = list(self.tables.values())[1:] + for child_table in child_tables: + criterion = self.join_child_tables( + criterion, + join_type=join_type, + child_table=child_table, + parent_table=parent_table, + ) return criterion, fields @@ -657,13 +730,16 @@ class Engine: ) -> MySQLQueryBuilder | PostgreSQLQueryBuilder: # Clean up state before each query self.tables = {} + self.joined_tables = {} self.linked_doctype = None self.fieldname = None - fields = self.set_fields(kwargs.get("field_objects") or fields, **kwargs) criterion = self.build_conditions(table, filters, **kwargs) - join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" - criterion, fields = self.join_(criterion=criterion, fields=fields, table=table, join=join) + fields = self.set_fields(table, fields, **kwargs) + join_type = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" + criterion, fields = self.join( + criterion=criterion, fields=fields, table=table, join_type=join_type + ) if isinstance(fields, (list, tuple)): query = criterion.select(*fields) diff --git a/frappe/database/utils.py b/frappe/database/utils.py index c1f70d388e..4ea039e5a7 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -18,6 +18,13 @@ QueryValues = tuple | list | dict | NoneType EmptyQueryValues = object() FallBackDateTimeStr = "0001-01-01 00:00:00.000000" +NestedSetHierarchy = ( + "ancestors of", + "descendants of", + "not ancestors of", + "not descendants of", +) + def is_query_type(query: str, query_type: str | tuple[str]) -> bool: return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index ad0bd549d8..bde27125f6 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -2,7 +2,6 @@ # License: MIT. See LICENSE import frappe -from frappe.translate import send_translations @frappe.whitelist() @@ -31,10 +30,6 @@ def getpage(): page = frappe.form_dict.get("name") doc = get(page) - # load translations - if frappe.lang != "en": - send_translations(frappe.get_lang_dict("page", page)) - frappe.response.docs.append(doc) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 1e515bbc47..5521d9583f 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -3,8 +3,10 @@ import frappe from frappe import _ +from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.model.document import Document from frappe.utils import cint +from frappe.utils.scheduler import is_scheduler_inactive class BulkUpdate(Document): @@ -44,8 +46,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): try: message = "" if action == "submit" and doc.docstatus.is_draft(): - doc.submit() - message = _("Submitting {0}").format(doctype) + if doc.meta.queue_in_background and not is_scheduler_inactive(): + queue_submission(doc, action) + message = _("Queuing {0} for Submission").format(doctype) + else: + doc.submit() + message = _("Submitting {0}").format(doctype) elif action == "cancel" and doc.docstatus.is_submitted(): doc.cancel() message = _("Cancelling {0}").format(doctype) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index d709f7b592..f43031c899 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -4,8 +4,10 @@ import json import frappe +from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.desk.form.load import run_onload from frappe.monitor import add_data_to_monitor +from frappe.utils.scheduler import is_scheduler_inactive @frappe.whitelist() @@ -16,8 +18,10 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] - if doc.docstatus == 1: + if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): + queue_submission(doc, action) + return doc.submit() else: doc.save() @@ -27,7 +31,6 @@ def savedocs(doc, action): send_updated_docs(doc) add_data_to_monitor(doctype=doc.doctype, action=action) - frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 1f8233510c..f83cc2d8ae 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -16,7 +16,6 @@ from frappe.model.utils import render_include from frappe.modules import get_module_path, scrub from frappe.monitor import add_data_to_monitor from frappe.permissions import get_role_permissions -from frappe.translate import send_translations from frappe.utils import ( cint, cstr, @@ -204,10 +203,6 @@ def get_script(report_name): if not script: script = "frappe.query_reports['%s']={}" % report_name - # load translations - if frappe.lang != "en": - send_translations(frappe.get_lang_dict("report", report_name)) - return { "script": render_include(script), "html_format": html_format, diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index e392816d35..d46f5c125a 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -91,7 +91,7 @@ def validate_args(data): def validate_fields(data): wildcard = update_wildcard_field_param(data) - for field in data.fields or []: + for field in list(data.fields or []): fieldname = extract_fieldname(field) if is_standard(fieldname): continue diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index f80d218bca..a25b6bda02 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -74,7 +74,7 @@ class TestNewsletterMixin: ).insert(ignore_if_duplicate=True) except Exception: frappe.db.rollback(save_point=savepoint) - frappe.db.update(doctype, email_filters, "unsubscribed", 0) + frappe.db.set_value(doctype, email_filters, "unsubscribed", 0) frappe.db.release_savepoint(savepoint) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 89b6df15d8..f5b60a9f3d 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -132,19 +132,18 @@ def oauth_access(email_account: str, service: str): if not service: frappe.throw(frappe._("No Service is selected. Please select one and try again!")) - doctype = "Email Account" - if service == "GMail": - return authorize_google_access(email_account, doctype) + return authorize_google_access(email_account) raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.") -def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None): +def authorize_google_access(email_account: str, code: str = None): """Facilitates google oauth for email. - This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url + This is invoked 2 times - first time when user clicks `Authorize API Access` for getting the authorization url and second time for setting the refresh and access token in db when google redirects back with oauth code.""" + doctype = "Email Account" oauth_obj = GoogleOAuth("mail") if not code: diff --git a/frappe/installer.py b/frappe/installer.py index 4f1755c2a0..2a6c29a17f 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -402,7 +402,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]: if not dry_run: if doctype.issingle: - frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True) + frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True, force=True) else: drop_doctypes.append(doctype.name) @@ -460,7 +460,7 @@ def _delete_doctypes(doctypes: list[str], dry_run: bool) -> None: for doctype in set(doctypes): print(f"* dropping Table for '{doctype}'...") if not dry_run: - frappe.delete_doc("DocType", doctype, ignore_on_trash=True) + frappe.delete_doc("DocType", doctype, ignore_on_trash=True, force=True) frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{doctype}`") diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index c6b3707b5c..8edf17a5d5 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -100,12 +100,7 @@ class BaseDocument: if d.get("doctype"): self.doctype = d["doctype"] - self._table_fieldnames = ( - d["_table_fieldnames"] # from cache - if "_table_fieldnames" in d - else {df.fieldname for df in self._get_table_fields()} - ) - + self._table_fieldnames = {df.fieldname for df in self._get_table_fields()} self.update(d) self.dont_update_if_missing = [] diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 8694afbc21..3e6b8ec753 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -13,7 +13,7 @@ import frappe.permissions import frappe.share from frappe import _ from frappe.core.doctype.server_script.server_script_utils import get_server_script_map -from frappe.database.utils import FallBackDateTimeStr +from frappe.database.utils import FallBackDateTimeStr, NestedSetHierarchy from frappe.model import optional_fields from frappe.model.meta import get_table_columns from frappe.model.utils.user_settings import get_user_settings, update_user_settings @@ -568,21 +568,14 @@ class DatabaseQuery: can_be_null = True # prepare in condition - if f.operator.lower() in ( - "ancestors of", - "descendants of", - "not ancestors of", - "not descendants of", - ): + if f.operator.lower() in NestedSetHierarchy: values = f.value or "" # TODO: handle list and tuple # if not isinstance(values, (list, tuple)): # values = values.split(",") - field = meta.get_field(f.fieldname) ref_doctype = field.options if field else f.doctype - lft, rgt = "", "" if f.value: lft, rgt = frappe.db.get_value(ref_doctype, f.value, ["lft", "rgt"]) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index d1120cc22d..5e8a12c345 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -92,6 +92,8 @@ def delete_doc( else: doc = frappe.get_doc(doctype, name) + if not (doc.custom or frappe.conf.developer_mode or frappe.flags.in_patch or force): + frappe.throw(_("Standard DocType can not be deleted.")) update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) diff --git a/frappe/model/document.py b/frappe/model/document.py index 2a2f924b00..f5f710a578 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -120,6 +120,10 @@ class Document(BaseDocument): # incorrect arguments. let's not proceed. raise ValueError("Illegal arguments") + @property + def is_locked(self): + return file_lock.lock_exists(self.get_signature()) + @staticmethod def whitelist(fn): """Decorator: Whitelist method to be called remotely via REST API.""" @@ -142,9 +146,14 @@ class Document(BaseDocument): self._fix_numeric_types() else: + get_value_kwargs = {"for_update": self.flags.for_update, "as_dict": True} + if not isinstance(self.name, (dict, list)): + get_value_kwargs["order_by"] = None + d = frappe.db.get_value( - self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update + doctype=self.doctype, filters=self.name, fieldname="*", **get_value_kwargs ) + if not d: frappe.throw( _("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError @@ -245,7 +254,6 @@ class Document(BaseDocument): self._set_defaults() self.set_user_and_timestamp() self.set_docstatus() - self.load_doc_before_save() self.check_if_latest() self._validate_links() self.check_permission("create") @@ -296,6 +304,10 @@ class Document(BaseDocument): follow_document(self.doctype, self.name, frappe.session.user) return self + def check_if_locked(self): + if self.creation and self.is_locked: + raise frappe.DocumentLockedError + def save(self, *args, **kwargs): """Wrapper for _save""" return self._save(*args, **kwargs) @@ -322,11 +334,11 @@ class Document(BaseDocument): if self.get("__islocal") or not self.get("name"): return self.insert() + self.check_if_locked() self.check_permission("write", "save") self.set_user_and_timestamp() self.set_docstatus() - self.load_doc_before_save() self.check_if_latest() self.set_parent_in_children() self.set_name_in_children() @@ -746,10 +758,13 @@ class Document(BaseDocument): Will also validate document transitions (Save > Submit > Cancel) calling `self.check_docstatus_transition`.""" - self._action = "save" - previous = self.get_doc_before_save() + self.load_doc_before_save(raise_exception=True) - if not previous or self.meta.get("is_virtual"): + self._action = "save" + previous = self._doc_before_save + + # previous is None for new document insert + if not previous: self.check_docstatus_transition(0) return @@ -1048,7 +1063,7 @@ class Document(BaseDocument): self.set_title_field() - def load_doc_before_save(self): + def load_doc_before_save(self, *, raise_exception: bool = False): """load existing document from db before saving""" self._doc_before_save = None @@ -1059,6 +1074,9 @@ class Document(BaseDocument): try: self._doc_before_save = frappe.get_doc(self.doctype, self.name, for_update=True) except frappe.DoesNotExistError: + if raise_exception: + raise + frappe.clear_last_message() def run_post_save_methods(self): diff --git a/frappe/model/naming.py b/frappe/model/naming.py index d9dc0ee48c..93be2204b4 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -235,13 +235,14 @@ def set_naming_from_document_naming_rule(doc): if doc.doctype in log_types: return - # ignore_ddl if naming is not yet bootstrapped - for d in frappe.get_all( + document_naming_rules = frappe.cache_manager.get_doctype_map( "Document Naming Rule", - dict(document_type=doc.doctype, disabled=0), + doc.doctype, + filters={"document_type": doc.doctype, "disabled": 0}, order_by="priority desc", - ignore_ddl=True, - ): + ) + + for d in document_naming_rules: frappe.get_cached_doc("Document Naming Rule", d.name).apply(doc) if doc.name: break @@ -277,7 +278,7 @@ def make_autoname(key="", doctype="", doc=""): DE/09/01/00001 where 09 is the year, 01 is the month and 00001 is the series """ if key == "hash": - return frappe.generate_hash(doctype, 10) + return frappe.generate_hash(length=10) series = NamingSeries(key) return series.generate_next_name(doc) diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index 923fbc1b3b..8338157996 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -1,12 +1,17 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import json +from typing import TYPE_CHECKING, Union import frappe from frappe import _ from frappe.model.docstatus import DocStatus from frappe.utils import cint +if TYPE_CHECKING: + from frappe.model.document import Document + from frappe.workflow.doctype.workflow.workflow import Workflow + class WorkflowStateError(frappe.ValidationError): pass @@ -32,20 +37,22 @@ def get_workflow_name(doctype): @frappe.whitelist() -def get_transitions(doc, workflow=None, raise_exception=False): +def get_transitions( + doc: Union["Document", str, dict], workflow: "Workflow" = None, raise_exception: bool = False +) -> list[dict]: """Return list of possible transitions for the given doc""" - doc = frappe.get_doc(frappe.parse_json(doc)) + from frappe.model.document import Document + + if not isinstance(doc, Document): + doc = frappe.get_doc(frappe.parse_json(doc)) + doc.load_from_db() if doc.is_new(): return [] - doc.load_from_db() + doc.check_permission("read") - frappe.has_permission(doc, "read", throw=True) - roles = frappe.get_roles() - - if not workflow: - workflow = get_workflow(doc.doctype) + workflow = workflow or get_workflow(doc.doctype) current_state = doc.get(workflow.workflow_state_field) if not current_state: @@ -55,11 +62,14 @@ def get_transitions(doc, workflow=None, raise_exception=False): frappe.throw(_("Workflow State not set"), WorkflowStateError) transitions = [] + roles = frappe.get_roles() + for transition in workflow.transitions: if transition.state == current_state and transition.allowed in roles: if not is_transition_condition_satisfied(transition, doc): continue transitions.append(transition.as_dict()) + return transitions @@ -79,7 +89,7 @@ def get_workflow_safe_globals(): ) -def is_transition_condition_satisfied(transition, doc): +def is_transition_condition_satisfied(transition, doc) -> bool: if not transition.condition: return True else: @@ -198,7 +208,7 @@ def validate_workflow(doc): ) -def get_workflow(doctype): +def get_workflow(doctype) -> "Workflow": return frappe.get_doc("Workflow", get_workflow_name(doctype)) diff --git a/frappe/patches/v11_0/remove_skip_for_doctype.py b/frappe/patches/v11_0/remove_skip_for_doctype.py index e7c1d71a0a..b3471ca4e8 100644 --- a/frappe/patches/v11_0/remove_skip_for_doctype.py +++ b/frappe/patches/v11_0/remove_skip_for_doctype.py @@ -60,7 +60,7 @@ def execute(): # Maintain sequence (name, user, allow, for_value, applicable_for, apply_to_all_doctypes, creation, modified) new_user_permissions_list.append( ( - frappe.generate_hash("", 10), + frappe.generate_hash(length=10), user_permission.user, user_permission.allow, user_permission.for_value, diff --git a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py index 1a369b4e12..7283760c23 100644 --- a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py +++ b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py @@ -27,7 +27,7 @@ def execute(): email_values.append( ( 1, - frappe.generate_hash(contact_detail.email_id, 10), + frappe.generate_hash(length=10), contact_detail.email_id, "email_ids", "Contact", @@ -44,7 +44,7 @@ def execute(): phone_values.append( ( phone_counter, - frappe.generate_hash(contact_detail.email_id, 10), + frappe.generate_hash(length=10), contact_detail.phone, "phone_nos", "Contact", @@ -63,7 +63,7 @@ def execute(): phone_values.append( ( phone_counter, - frappe.generate_hash(contact_detail.email_id, 10), + frappe.generate_hash(length=10), contact_detail.mobile_no, "phone_nos", "Contact", diff --git a/frappe/patches/v12_0/setup_tags.py b/frappe/patches/v12_0/setup_tags.py index 6bff8d3dac..cb0d46a45d 100644 --- a/frappe/patches/v12_0/setup_tags.py +++ b/frappe/patches/v12_0/setup_tags.py @@ -28,7 +28,7 @@ def execute(): tag_list.append((tag.strip(), time, time, "Administrator")) - tag_link_name = frappe.generate_hash(_user_tags.name + tag.strip() + doctype.name, 10) + tag_link_name = frappe.generate_hash(length=10) tag_links.append( (tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, "Administrator") ) diff --git a/frappe/patches/v13_0/queryreport_columns.py b/frappe/patches/v13_0/queryreport_columns.py index 3081823db6..e9176952d4 100644 --- a/frappe/patches/v13_0/queryreport_columns.py +++ b/frappe/patches/v13_0/queryreport_columns.py @@ -15,4 +15,4 @@ def execute(): if isinstance(data, list): # double escape braces jstr = f'{{"columns":{jstr}}}' - frappe.db.update("Report", record["name"], "json", jstr) + frappe.db.set_value("Report", record["name"], "json", jstr) diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py index 4b0a58c2d6..cff2b583ce 100644 --- a/frappe/patches/v14_0/remove_db_aggregation.py +++ b/frappe/patches/v14_0/remove_db_aggregation.py @@ -32,4 +32,4 @@ def execute(): for agg in ["avg", "max", "min", "sum"]: script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script) - frappe.db.update("Server Script", name, "script", script) + frappe.db.set_value("Server Script", name, "script", script) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8c642a73f0..4f119f2551 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -450,6 +450,7 @@ frappe.ui.form.Form = class FrappeForm { .toggleClass("cancelled-form", this.doc.docstatus === 2); this.show_conflict_message(); + this.show_submission_queue_banner(); if (frappe.boot.read_only) { this.disable_form(); @@ -2036,6 +2037,83 @@ frappe.ui.form.Form = class FrappeForm { .filter((user) => !["Administrator", frappe.session.user].includes(user)) .filter(Boolean); } + + show_submission_queue_banner() { + let wrapper = this.layout.wrapper.find(".submission-queue-banner"); + + if ( + !( + this.meta.is_submittable && + this.meta.queue_in_background && + !this.doc.__islocal && + this.doc.docstatus === 0 + ) + ) { + if (wrapper.length) { + wrapper.hide(); + wrapper.html(""); + } + + return; + } + + if (!wrapper.length) { + wrapper = $('
'); + this.layout.wrapper.prepend(wrapper); + } + + frappe + .call({ + method: "frappe.core.doctype.submission_queue.submission_queue.get_latest_submissions", + args: { doctype: this.doctype, docname: this.docname }, + }) + .then((r) => { + if (r.message.latest_submission) { + // if we are here that means some submission(s) were queued and are in queued/failed state + let col_width = 4; + let failed_link = ""; + let submission_label = __("Previous Submission"); + + if (r.message.latest_failed_submission) { + if (r.message.latest_failed_submission !== r.message.latest_submission) { + col_width = 3; + failed_link = `
+ ${__( + "Previous Falied Submission" + )} +
`; + } else { + submission_label = __("Previous Falied Submission"); + } + } + + let html = ` +
+
+ ${__("Submission Status:")} +
+ + ${failed_link} + +
+ `; + + wrapper.show(); + wrapper.html(html); + } else { + wrapper.hide(); + wrapper.html(""); + } + }); + } }; frappe.validated = 0; diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 4a30ad68e0..f9f09187bc 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -944,19 +944,21 @@ export default class GridRow { vertical = false; horizontal = false; }) - .on("click", function () { + .on("click", function (event) { if (frappe.ui.form.editable_row !== me) { var out = me.toggle_editable_row(); } var col = this; let first_input_field = $(col).find('input[type="Text"]:first'); - - first_input_field.length && on_input_focus(first_input_field); - first_input_field.trigger("focus"); - first_input_field.one("blur", () => (input_in_focus = false)); - first_input_field.data("fieldtype") == "Date" && handle_date_picker(); + if (event.pointerType == "touch") { + first_input_field.length && on_input_focus(first_input_field); + + first_input_field.one("blur", () => (input_in_focus = false)); + + first_input_field.data("fieldtype") == "Date" && handle_date_picker(); + } return out; }); diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 66ff5107a1..52d0026e37 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1179,7 +1179,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { this.$result.on("click", ".list-row, .image-view-header, .file-header", (e) => { const $target = $(e.target); // tick checkbox if Ctrl/Meta key is pressed - if (e.ctrlKey || (e.metaKey && !$target.is("a"))) { + if ((e.ctrlKey || e.metaKey) && !$target.is("a")) { const $list_row = $(e.currentTarget); const $check = $list_row.find(".list-row-checkbox"); $check.prop("checked", !$check.prop("checked")); diff --git a/frappe/public/js/frappe/list/list_view_select.js b/frappe/public/js/frappe/list/list_view_select.js index 0e7033cf9a..5265ace340 100644 --- a/frappe/public/js/frappe/list/list_view_select.js +++ b/frappe/public/js/frappe/list/list_view_select.js @@ -315,7 +315,7 @@ frappe.views.ListViewSelect = class ListViewSelect { accounts.forEach((account) => { let email_account = account.email_id == "All Accounts" ? "All Accounts" : account.email_account; - let route = `/app/communication/inbox/${email_account}`; + let route = `/app/communication/view/inbox/${email_account}`; let display_name = ["All Accounts", "Sent Mail", "Spam", "Trash"].includes( account.email_id ) diff --git a/frappe/public/js/frappe/ui/keyboard.js b/frappe/public/js/frappe/ui/keyboard.js index 14418528a6..2e3bd95616 100644 --- a/frappe/public/js/frappe/ui/keyboard.js +++ b/frappe/public/js/frappe/ui/keyboard.js @@ -331,7 +331,7 @@ function close_grid_and_dialog() { } // close open dialog - if (cur_dialog && !cur_dialog.no_cancel_flag) { + if (cur_dialog && !cur_dialog.no_cancel_flag && !cur_dialog.static) { cur_dialog.cancel(); return false; } diff --git a/frappe/public/js/frappe/ui/page.js b/frappe/public/js/frappe/ui/page.js index 0c15eef774..e381f332a9 100644 --- a/frappe/public/js/frappe/ui/page.js +++ b/frappe/public/js/frappe/ui/page.js @@ -327,13 +327,14 @@ frappe.ui.Page = class Page { //--- Menu --// - add_menu_item(label, click, standard, shortcut) { + add_menu_item(label, click, standard, shortcut, show_parent) { return this.add_dropdown_item({ label, click, standard, parent: this.menu, shortcut, + show_parent, }); } @@ -424,7 +425,7 @@ frappe.ui.Page = class Page { icon = null, }) { if (show_parent) { - parent.parent().removeClass("hide"); + parent.parent().removeClass("hide hidden-xl"); } let $link = this.is_in_group_button_dropdown(parent, "li > a.grey-link > span", label); @@ -600,6 +601,14 @@ frappe.ui.Page = class Page { let response = action(); me.btn_disable_enable(btn, response); }; + // Add actions as menu item in Mobile View + let menu_item_label = group ? `${group} > ${label}` : label; + let menu_item = this.add_menu_item(menu_item_label, _action, false, false, false); + menu_item.parent().addClass("hidden-xl"); + if (this.menu_btn_group.hasClass("hide")) { + this.menu_btn_group.removeClass("hide").addClass("hidden-xl"); + } + if (group) { var $group = this.get_or_add_inner_group_button(group); $(this.inner_toolbar).removeClass("hide"); diff --git a/frappe/public/js/frappe/ui/toolbar/about.js b/frappe/public/js/frappe/ui/toolbar/about.js index e23706eff1..69cbbfaba0 100644 --- a/frappe/public/js/frappe/ui/toolbar/about.js +++ b/frappe/public/js/frappe/ui/toolbar/about.js @@ -19,6 +19,8 @@ frappe.ui.misc.about = function () { Facebook: https://facebook.com/erpnext

Twitter: https://twitter.com/erpnext

+

+ YouTube: https://www.youtube.com/@erpnextofficial


${__("Installed Apps")}

${__("Loading versions...")}
diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 3d17e8a8f6..cf5b619c37 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1383,7 +1383,7 @@ Object.assign(frappe.utils, { : ""; return $(`
- ${summary.label} + ${__(summary.label)}
${value}
`); }, diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index 5236c21ac2..74560cebbc 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -40,7 +40,6 @@ frappe.breadcrumbs = { type: type, }; } - this.all[frappe.breadcrumbs.current_page()] = obj; this.update(); }, @@ -137,13 +136,13 @@ frappe.breadcrumbs = { const doctype_meta = frappe.get_doc("DocType", doctype); if ( (doctype === "User" && !frappe.user.has_role("System Manager")) || - (doctype_meta && doctype_meta.issingle) + doctype_meta?.issingle ) { // no user listview for non-system managers and single doctypes } else { let route; const doctype_route = frappe.router.slug(frappe.router.doctype_layout || doctype); - if (doctype_meta.is_tree) { + if (doctype_meta?.is_tree) { let view = frappe.model.user_settings[doctype].last_view || "Tree"; route = `${doctype_route}/view/${view}`; } else { diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index f71391a1c3..7f20a69efc 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1067,7 +1067,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { { fieldname: "sb_1", fieldtype: "Section Break", - label: "Y axis", + label: "Y Axis", }, { fieldname: "y_axis_fields", diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index 60d0a1208d..5a7b06221f 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -47,6 +47,8 @@ class Base: class MariaDB(Base, MySQLQuery): Field = terms.Field + _BuilderClasss = MySQLQueryBuilder + @classmethod def _builder(cls, *args, **kwargs) -> "MySQLQueryBuilder": return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) @@ -70,6 +72,8 @@ class Postgres(Base, PostgreSQLQuery): # they are two different objects. The quick fix used here is to replace the # Field names in the "Field" function. + _BuilderClasss = PostgreSQLQueryBuilder + @classmethod def _builder(cls, *args, **kwargs) -> "PostgreSQLQueryBuilder": return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index f0130ca813..be0403a291 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -12,6 +12,16 @@ from frappe.query_builder.terms import NamedParameterWrapper from .builder import MariaDB, Postgres +class PseudoColumnMapper(PseudoColumn): + def __init__(self, name: str) -> None: + super().__init__(name) + + def get_sql(self, **kwargs): + if frappe.db.db_type == "postgres": + self.name = self.name.replace("`", '"') + return self.name + + class db_type_is(Enum): MARIADB = "mariadb" POSTGRES = "postgres" @@ -102,8 +112,7 @@ def patch_query_execute(): raise frappe.PermissionError("Only SELECT SQL allowed in scripting") return query, param_collector.get_parameters() - query_class = get_attr(str(frappe.qb).split("'")[1]) - builder_class = get_type_hints(query_class._builder).get("return") + builder_class = frappe.qb._BuilderClasss if not builder_class: raise BuilderIdentificationFailed diff --git a/frappe/templates/includes/navbar/navbar_items.html b/frappe/templates/includes/navbar/navbar_items.html index 8a10751441..99a40a02a0 100644 --- a/frappe/templates/includes/navbar/navbar_items.html +++ b/frappe/templates/includes/navbar/navbar_items.html @@ -3,7 +3,7 @@ {% if parent %} -{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%} +{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}