From f62d1540cf4a97b9da1c54fb94b75f32a51f5015 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 24 Sep 2022 14:00:55 +0530 Subject: [PATCH 001/100] feat: Background submissions for submittable doctypes --- frappe/core/doctype/doctype/doctype.json | 11 +++++++++-- frappe/desk/form/save.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index bfa91cea75..4e6c4bc80f 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -85,7 +85,8 @@ "website_search_field", "advanced", "engine", - "migration_hash" + "migration_hash", + "bg_submit" ], "fields": [ { @@ -605,6 +606,12 @@ "fieldname": "make_attachments_public", "fieldtype": "Check", "label": "Make Attachments Public by Default" + }, + { + "default": "0", + "fieldname": "bg_submit", + "fieldtype": "Check", + "label": "BgSubmit" } ], "icon": "fa fa-bolt", @@ -687,7 +694,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-08-24 06:42:27.779699", + "modified": "2022-09-24 11:10:26.020900", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index f3e7b6294f..78396f666b 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -17,9 +17,15 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - doc.submit() + if doc.bg_submit and doc.is_submittable: + doc.queue_action("submit", timeout=4000) + else: + doc.submit() else: - doc.save() + if doc.bg_submit and doc.is_submittable: + doc.queue_action("save", timeout=4000) + else: + doc.save() # update recent documents run_onload(doc) From 8b1a01e4b7e914c5b20a7d2fe3491851c0732e7a Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 30 Sep 2022 19:25:06 +0530 Subject: [PATCH 002/100] refactor: changed field name & removed queueing from save --- frappe/core/doctype/doctype/doctype.json | 8 ++++---- frappe/desk/form/save.py | 8 ++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 4e6c4bc80f..15671bc2b8 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -86,7 +86,7 @@ "advanced", "engine", "migration_hash", - "bg_submit" + "submit_in_background" ], "fields": [ { @@ -609,9 +609,9 @@ }, { "default": "0", - "fieldname": "bg_submit", + "fieldname": "submit_in_background", "fieldtype": "Check", - "label": "BgSubmit" + "label": "Submit in background" } ], "icon": "fa fa-bolt", @@ -694,7 +694,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-09-24 11:10:26.020900", + "modified": "2022-09-30 19:18:02.789866", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index d8204337de..71ec91e6ce 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -16,17 +16,13 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] - if doc.docstatus == 1: - if doc.bg_submit and doc.is_submittable: + if doc.meta.submit_in_background and doc.meta.is_submittable: doc.queue_action("submit", timeout=4000) else: doc.submit() else: - if doc.bg_submit and doc.is_submittable: - doc.queue_action("save", timeout=4000) - else: - doc.save() + doc.save() # update recent documents run_onload(doc) From b787a49428d8c0a1aff2ad76e519a6d736b0cf2e Mon Sep 17 00:00:00 2001 From: Aradhya Tripathi <67282231+Aradhya-Tripathi@users.noreply.github.com> Date: Mon, 3 Oct 2022 13:46:06 +0530 Subject: [PATCH 003/100] refactor: removed excess checks Co-authored-by: Ritwik Puri --- frappe/desk/form/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 71ec91e6ce..3cdd4e6f01 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -17,7 +17,7 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if doc.meta.submit_in_background and doc.meta.is_submittable: + if doc.meta.submit_in_background: doc.queue_action("submit", timeout=4000) else: doc.submit() From eae73ae5d45414f7e8b4dce0326abae8b41c9e26 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 3 Oct 2022 17:36:33 +0530 Subject: [PATCH 004/100] feat: log reports for queued submits --- frappe/core/doctype/doctype/doctype.json | 6 +- frappe/core/doctype/queued_submit/__init__.py | 0 .../doctype/queued_submit/queued_submit.js | 8 ++ .../doctype/queued_submit/queued_submit.json | 80 +++++++++++++++++++ .../doctype/queued_submit/queued_submit.py | 8 ++ .../queued_submit/test_queued_submit.py | 9 +++ frappe/desk/form/save.py | 4 +- frappe/model/document.py | 23 ++++++ frappe/utils/background_jobs.py | 1 + 9 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 frappe/core/doctype/queued_submit/__init__.py create mode 100644 frappe/core/doctype/queued_submit/queued_submit.js create mode 100644 frappe/core/doctype/queued_submit/queued_submit.json create mode 100644 frappe/core/doctype/queued_submit/queued_submit.py create mode 100644 frappe/core/doctype/queued_submit/test_queued_submit.py diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 15671bc2b8..b43a9ec3ae 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -58,6 +58,7 @@ "icon", "color", "show_preview_popup", + "submit_in_background", "show_name_in_global_search", "email_settings_sb", "default_email_template", @@ -85,8 +86,7 @@ "website_search_field", "advanced", "engine", - "migration_hash", - "submit_in_background" + "migration_hash" ], "fields": [ { @@ -694,7 +694,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-09-30 19:18:02.789866", + "modified": "2022-10-03 16:14:46.432770", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/core/doctype/queued_submit/__init__.py b/frappe/core/doctype/queued_submit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/queued_submit/queued_submit.js b/frappe/core/doctype/queued_submit/queued_submit.js new file mode 100644 index 0000000000..fec6615884 --- /dev/null +++ b/frappe/core/doctype/queued_submit/queued_submit.js @@ -0,0 +1,8 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Queued Submit', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/queued_submit/queued_submit.json b/frappe/core/doctype/queued_submit/queued_submit.json new file mode 100644 index 0000000000..9cd2dde1b5 --- /dev/null +++ b/frappe/core/doctype/queued_submit/queued_submit.json @@ -0,0 +1,80 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2022-10-03 17:19:13.116028", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "title", + "job_id", + "state", + "start_time", + "created_by", + "error" + ], + "fields": [ + { + "fieldname": "title", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Title", + "reqd": 1 + }, + { + "fieldname": "job_id", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Job Id", + "reqd": 1 + }, + { + "fieldname": "state", + "fieldtype": "Data", + "label": "State" + }, + { + "fieldname": "start_time", + "fieldtype": "Datetime", + "in_list_view": 1, + "label": "Start Time", + "reqd": 1 + }, + { + "fieldname": "created_by", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Created By", + "reqd": 1 + }, + { + "fieldname": "error", + "fieldtype": "Text", + "label": "Error" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2022-10-03 17:28:12.081732", + "modified_by": "Administrator", + "module": "Core", + "name": "Queued Submit", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/queued_submit/queued_submit.py b/frappe/core/doctype/queued_submit/queued_submit.py new file mode 100644 index 0000000000..65ee12a0c3 --- /dev/null +++ b/frappe/core/doctype/queued_submit/queued_submit.py @@ -0,0 +1,8 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class QueuedSubmit(Document): + pass diff --git a/frappe/core/doctype/queued_submit/test_queued_submit.py b/frappe/core/doctype/queued_submit/test_queued_submit.py new file mode 100644 index 0000000000..86c4c803ca --- /dev/null +++ b/frappe/core/doctype/queued_submit/test_queued_submit.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestQueuedSubmit(FrappeTestCase): + pass diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 3cdd4e6f01..05484664a5 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -18,9 +18,10 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - doc.queue_action("submit", timeout=4000) + doc.submit_in_background() else: doc.submit() + else: doc.save() @@ -29,7 +30,6 @@ def savedocs(doc, action): send_updated_docs(doc) add_data_to_monitor(doctype=doc.doctype, action=action) - frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) diff --git a/frappe/model/document.py b/frappe/model/document.py index aa55eac30a..489cb56a19 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,5 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from datetime import datetime import hashlib import json import time @@ -984,6 +985,28 @@ class Document(BaseDocument): elif alert.event == "Method" and method == alert.method: _evaluate_alert(alert) + def submit_in_background(self): + job = self.queue_action("_submit_in_background") + doc = frappe.new_doc("Queued Submit") + doc.title = self.name + doc.state = "Queued" + doc.start_time = datetime.now() + doc.job_id = job.id + doc.created_by = frappe.session.user + doc.insert() + + def _submit_in_background(self): + try: + self.submit() + doc = frappe.get_doc("Queued Submit", self.title) + doc.state = "Submitted" + doc.insert() + except Exception as e: + doc = frappe.get_doc("Queued Submit", self.title) + doc.state = "Failed" + doc.error = str(e) + doc.insert() + @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index e88cd75efb..0834a403da 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -9,6 +9,7 @@ from uuid import uuid4 import redis from redis.exceptions import BusyLoadingError, ConnectionError from rq import Connection, Queue, Worker +from rq.registry import FailedJobRegistry from rq.command import send_stop_job_command from rq.logutils import setup_loghandlers from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed From 9cc826541fe2b27892adfe6c5870c8890b1a76a2 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 02:41:14 +0530 Subject: [PATCH 005/100] fix: fixed functionality --- frappe/core/doctype/queued_submit/__init__.py | 0 .../doctype/queued_submit/queued_submit.js | 8 -- .../doctype/queued_submit/queued_submit.json | 80 ------------------- .../doctype/queued_submit/queued_submit.py | 8 -- .../queued_submit/test_queued_submit.py | 9 --- frappe/model/document.py | 19 ++--- frappe/utils/background_jobs.py | 1 - 7 files changed, 8 insertions(+), 117 deletions(-) delete mode 100644 frappe/core/doctype/queued_submit/__init__.py delete mode 100644 frappe/core/doctype/queued_submit/queued_submit.js delete mode 100644 frappe/core/doctype/queued_submit/queued_submit.json delete mode 100644 frappe/core/doctype/queued_submit/queued_submit.py delete mode 100644 frappe/core/doctype/queued_submit/test_queued_submit.py diff --git a/frappe/core/doctype/queued_submit/__init__.py b/frappe/core/doctype/queued_submit/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/core/doctype/queued_submit/queued_submit.js b/frappe/core/doctype/queued_submit/queued_submit.js deleted file mode 100644 index fec6615884..0000000000 --- a/frappe/core/doctype/queued_submit/queued_submit.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2022, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Queued Submit', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/queued_submit/queued_submit.json b/frappe/core/doctype/queued_submit/queued_submit.json deleted file mode 100644 index 9cd2dde1b5..0000000000 --- a/frappe/core/doctype/queued_submit/queued_submit.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "actions": [], - "allow_rename": 1, - "creation": "2022-10-03 17:19:13.116028", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "title", - "job_id", - "state", - "start_time", - "created_by", - "error" - ], - "fields": [ - { - "fieldname": "title", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Title", - "reqd": 1 - }, - { - "fieldname": "job_id", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Job Id", - "reqd": 1 - }, - { - "fieldname": "state", - "fieldtype": "Data", - "label": "State" - }, - { - "fieldname": "start_time", - "fieldtype": "Datetime", - "in_list_view": 1, - "label": "Start Time", - "reqd": 1 - }, - { - "fieldname": "created_by", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Created By", - "reqd": 1 - }, - { - "fieldname": "error", - "fieldtype": "Text", - "label": "Error" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2022-10-03 17:28:12.081732", - "modified_by": "Administrator", - "module": "Core", - "name": "Queued Submit", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "states": [] -} \ No newline at end of file diff --git a/frappe/core/doctype/queued_submit/queued_submit.py b/frappe/core/doctype/queued_submit/queued_submit.py deleted file mode 100644 index 65ee12a0c3..0000000000 --- a/frappe/core/doctype/queued_submit/queued_submit.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2022, Frappe Technologies and contributors -# For license information, please see license.txt - -# import frappe -from frappe.model.document import Document - -class QueuedSubmit(Document): - pass diff --git a/frappe/core/doctype/queued_submit/test_queued_submit.py b/frappe/core/doctype/queued_submit/test_queued_submit.py deleted file mode 100644 index 86c4c803ca..0000000000 --- a/frappe/core/doctype/queued_submit/test_queued_submit.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2022, Frappe Technologies and Contributors -# See license.txt - -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestQueuedSubmit(FrappeTestCase): - pass diff --git a/frappe/model/document.py b/frappe/model/document.py index 489cb56a19..4e2cac89c1 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from datetime import datetime import hashlib import json import time +from datetime import datetime from werkzeug.exceptions import NotFound @@ -987,25 +987,22 @@ class Document(BaseDocument): def submit_in_background(self): job = self.queue_action("_submit_in_background") - doc = frappe.new_doc("Queued Submit") - doc.title = self.name + doc = frappe.new_doc("Submission Queue") doc.state = "Queued" doc.start_time = datetime.now() - doc.job_id = job.id doc.created_by = frappe.session.user + doc.name = self.doctype + str(self.name) + doc.job_id = job.id doc.insert() def _submit_in_background(self): try: self.submit() - doc = frappe.get_doc("Queued Submit", self.title) - doc.state = "Submitted" - doc.insert() + frappe.db.set_value("Submission Queue", self.doctype + str(self.name), {"state": "Submitted"}) except Exception as e: - doc = frappe.get_doc("Queued Submit", self.title) - doc.state = "Failed" - doc.error = str(e) - doc.insert() + frappe.db.set_value( + "Submission Queue", self.doctype + str(self.name), {"state": "Failed", "error": e} + ) @whitelist.__func__ def _submit(self): diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 0834a403da..e88cd75efb 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -9,7 +9,6 @@ from uuid import uuid4 import redis from redis.exceptions import BusyLoadingError, ConnectionError from rq import Connection, Queue, Worker -from rq.registry import FailedJobRegistry from rq.command import send_stop_job_command from rq.logutils import setup_loghandlers from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed From 198bc39085979dda260c292621a29eb5bcbc787c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 17:17:45 +0530 Subject: [PATCH 006/100] refactor: moved submission from document class --- frappe/core/doctype/report/report.js | 1 - .../core/doctype/submission_queue/__init__.py | 0 .../submission_queue/submission_queue.js | 7 ++ .../submission_queue/submission_queue.json | 78 +++++++++++++++++++ .../submission_queue/submission_queue.py | 44 +++++++++++ .../submission_queue/test_submission_queue.py | 9 +++ frappe/desk/form/save.py | 8 +- frappe/model/document.py | 19 ----- 8 files changed, 143 insertions(+), 23 deletions(-) create mode 100644 frappe/core/doctype/submission_queue/__init__.py create mode 100644 frappe/core/doctype/submission_queue/submission_queue.js create mode 100644 frappe/core/doctype/submission_queue/submission_queue.json create mode 100644 frappe/core/doctype/submission_queue/submission_queue.py create mode 100644 frappe/core/doctype/submission_queue/test_submission_queue.py diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index 0e0bfeea9a..9850dbf98f 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -31,7 +31,6 @@ frappe.ui.form.on("Report", { ); } - if (doc.is_standard === "Yes" && frm.perm[0].write) { frm.add_custom_button( doc.disabled ? __("Enable Report") : __("Disable Report"), diff --git a/frappe/core/doctype/submission_queue/__init__.py b/frappe/core/doctype/submission_queue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js new file mode 100644 index 0000000000..df0833c004 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -0,0 +1,7 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Submission Queue", { + // refresh: function(frm) { + // } +}); diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json new file mode 100644 index 0000000000..638d28d3fe --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -0,0 +1,78 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2022-10-04 00:41:00.028163", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "state", + "start_time", + "created_by", + "job_id", + "ref_doctype", + "ref_docname" + ], + "fields": [ + { + "fieldname": "state", + "fieldtype": "Select", + "label": "State", + "options": "Submitted\nQueued\nFailed", + "read_only": 1 + }, + { + "fieldname": "start_time", + "fieldtype": "Datetime", + "label": "Start time", + "read_only": 1 + }, + { + "fieldname": "created_by", + "fieldtype": "Data", + "label": "Created By", + "read_only": 1 + }, + { + "fieldname": "job_id", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Job Id", + "read_only": 1 + }, + { + "fieldname": "ref_doctype", + "fieldtype": "Link", + "label": "Reference DocType", + "options": "DocType" + }, + { + "fieldname": "ref_docname", + "fieldtype": "Dynamic Link", + "label": "Reference Docname", + "options": "ref_doctype" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2022-10-04 16:16:42.962361", + "modified_by": "Administrator", + "module": "Core", + "name": "Submission Queue", + "naming_rule": "Random", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py new file mode 100644 index 0000000000..4f4a9a8ca4 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -0,0 +1,44 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +from datetime import datetime + +import frappe +from frappe import _ + +# import frappe +from frappe.model.document import Document +from frappe.utils.background_jobs import enqueue + + +class SubmissionQueue(Document): + ... + + +def submit_in_background(doc: Document): + try: + doc.lock() + except frappe.DocumentLockedError: + frappe.throw( + _("This document is currently queued for execution. Please try again"), + title=_("Document Queued"), + exc=frappe.DocumentLockedError, + ) + new_queue = frappe.new_doc("Submission Queue") + new_queue.state = "Queued" + new_queue.start_time = datetime.now() + new_queue.created_by = frappe.session.user + new_queue.ref_doctype = doc.doctype + new_queue.ref_docname = doc.name + new_queue.insert() + job = enqueue(_submit_in_background, name=new_queue.name, doc=doc) + new_queue.job_id = job.id + new_queue.save() + + +def _submit_in_background(name: str, doc: Document): + try: + doc.submit() + frappe.db.set_value("Submission Queue", name, {"state": "Submitted"}) + except Exception as e: + frappe.db.set_value("Submission Queue", name, {"state": "Failed", "error": e}) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py new file mode 100644 index 0000000000..d7547983a2 --- /dev/null +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestSubmissionQueue(FrappeTestCase): + pass diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 05484664a5..5944656a13 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -4,6 +4,7 @@ import json import frappe +from frappe.core.doctype.submission_queue.submission_queue import submit_in_background from frappe.desk.form.load import run_onload from frappe.monitor import add_data_to_monitor @@ -18,10 +19,10 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - doc.submit_in_background() + submit_in_background(doc) + frappe.msgprint(frappe._("Queued"), indicator="green", alert=True) else: doc.submit() - else: doc.save() @@ -30,7 +31,8 @@ def savedocs(doc, action): send_updated_docs(doc) add_data_to_monitor(doctype=doc.doctype, action=action) - frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) + if not doc.meta.submit_in_background: + frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) @frappe.whitelist() diff --git a/frappe/model/document.py b/frappe/model/document.py index 4e2cac89c1..ca27a830d3 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -985,25 +985,6 @@ class Document(BaseDocument): elif alert.event == "Method" and method == alert.method: _evaluate_alert(alert) - def submit_in_background(self): - job = self.queue_action("_submit_in_background") - doc = frappe.new_doc("Submission Queue") - doc.state = "Queued" - doc.start_time = datetime.now() - doc.created_by = frappe.session.user - doc.name = self.doctype + str(self.name) - doc.job_id = job.id - doc.insert() - - def _submit_in_background(self): - try: - self.submit() - frappe.db.set_value("Submission Queue", self.doctype + str(self.name), {"state": "Submitted"}) - except Exception as e: - frappe.db.set_value( - "Submission Queue", self.doctype + str(self.name), {"state": "Failed", "error": e} - ) - @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" From 33acaf7efdb7df363bb59e2391550d4632f58bf1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 17:24:27 +0530 Subject: [PATCH 007/100] fix: unlock document when action is executed --- frappe/core/doctype/submission_queue/submission_queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4f4a9a8ca4..38f0262fce 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -37,6 +37,7 @@ def submit_in_background(doc: Document): def _submit_in_background(name: str, doc: Document): + doc.unlock() try: doc.submit() frappe.db.set_value("Submission Queue", name, {"state": "Submitted"}) From 4bdffe73ddd3f0e74ceac7dbece6cf8e86983230 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 23:06:46 +0530 Subject: [PATCH 008/100] refactor: moved background submit to Submission Queue DocType Co-authored-by: phot0n ritwikpuri5678@gmail.com --- .../submission_queue/submission_queue.json | 60 +++++++++++--- .../submission_queue/submission_queue.py | 82 ++++++++++++------- frappe/desk/form/save.py | 10 +-- 3 files changed, 104 insertions(+), 48 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 638d28d3fe..f2972c5927 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -6,21 +6,17 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ - "state", + "status", "start_time", "created_by", "job_id", + "column_break_5", "ref_doctype", - "ref_docname" + "ref_docname", + "section_break_8", + "message" ], "fields": [ - { - "fieldname": "state", - "fieldtype": "Select", - "label": "State", - "options": "Submitted\nQueued\nFailed", - "read_only": 1 - }, { "fieldname": "start_time", "fieldtype": "Datetime", @@ -44,18 +40,43 @@ "fieldname": "ref_doctype", "fieldtype": "Link", "label": "Reference DocType", - "options": "DocType" + "options": "DocType", + "read_only": 1 }, { "fieldname": "ref_docname", "fieldtype": "Dynamic Link", "label": "Reference Docname", - "options": "ref_doctype" + "options": "ref_doctype", + "read_only": 1 + }, + { + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "in_list_view": 1, + "label": "Status", + "options": "Queued\nCompleted\nFailed", + "read_only": 1 + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "message", + "fieldtype": "Text", + "label": "Message", + "read_only": 1 + }, + { + "fieldname": "section_break_8", + "fieldtype": "Section Break" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-04 16:16:42.962361", + "modified": "2022-10-04 18:41:08.495515", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -74,5 +95,18 @@ ], "sort_field": "modified", "sort_order": "DESC", - "states": [] + "states": [ + { + "color": "Blue", + "title": "Queued" + }, + { + "color": "Red", + "title": "Failed" + }, + { + "color": "Green", + "title": "Completed" + } + ] } \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 38f0262fce..621de6357b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -5,41 +5,63 @@ from datetime import datetime import frappe from frappe import _ - -# import frappe +from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document -from frappe.utils.background_jobs import enqueue class SubmissionQueue(Document): - ... + def insert(self, to_be_queued_doc: Document, action: str): + self.to_be_queued_doc = to_be_queued_doc + self.action_for_queuing = action + super().insert() - -def submit_in_background(doc: Document): - try: - doc.lock() - except frappe.DocumentLockedError: - frappe.throw( - _("This document is currently queued for execution. Please try again"), - title=_("Document Queued"), - exc=frappe.DocumentLockedError, + def after_insert(self): + job = self.queue_action( + "queue_in_background", + to_be_queued_doc=self.to_be_queued_doc, + action_for_queuing=self.action_for_queuing, ) - new_queue = frappe.new_doc("Submission Queue") - new_queue.state = "Queued" - new_queue.start_time = datetime.now() - new_queue.created_by = frappe.session.user - new_queue.ref_doctype = doc.doctype - new_queue.ref_docname = doc.name - new_queue.insert() - job = enqueue(_submit_in_background, name=new_queue.name, doc=doc) - new_queue.job_id = job.id - new_queue.save() + frappe.db.set_value(self.doctype, self.name, {"job_id": job.id}, update_modified=False) -def _submit_in_background(name: str, doc: Document): - doc.unlock() - try: - doc.submit() - frappe.db.set_value("Submission Queue", name, {"state": "Submitted"}) - except Exception as e: - frappe.db.set_value("Submission Queue", name, {"state": "Failed", "error": e}) + def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): + _action = action_for_queuing.lower() + + if _action == "update": + _action = "submit" + + try: + getattr(to_be_queued_doc, _action)() + values = {"status": "Completed"} + except Exception as e: + values = {"status": "Failed", "message": str(e)} + + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + notify(name=self.name) + + +def notify(name: str): + notification_doc = { + "type": "Notification", + "document_type": "Submission Queue", + "document_name": name, + "subject": "Job Queued", + "from_user": frappe.session.user, + "email_content": "Hello", + } + + mention = frappe.db.get_value("User", filters=frappe.db.get_value( + "Submission Queue", filters=name, fieldname="created_by" + ), fieldname="email") + if mention: + enqueue_create_notification([mention], notification_doc) + + +def queue_submission(doc: Document, action: str): + queue = frappe.new_doc("Submission Queue") + queue.state = "Queued" + queue.start_time = datetime.now() + queue.created_by = frappe.session.user + queue.ref_doctype = doc.doctype + queue.ref_docname = doc.name + queue.insert(doc, action) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 5944656a13..d6a7e70b4d 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -4,7 +4,7 @@ import json import frappe -from frappe.core.doctype.submission_queue.submission_queue import submit_in_background +from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.desk.form.load import run_onload from frappe.monitor import add_data_to_monitor @@ -19,8 +19,9 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - submit_in_background(doc) - frappe.msgprint(frappe._("Queued"), indicator="green", alert=True) + queue_submission(doc, action) + frappe.msgprint(frappe._("Queued for Submission"), indicator="green", alert=True) + return else: doc.submit() else: @@ -31,8 +32,7 @@ def savedocs(doc, action): send_updated_docs(doc) add_data_to_monitor(doctype=doc.doctype, action=action) - if not doc.meta.submit_in_background: - frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) + frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) @frappe.whitelist() From 1230b78337c06a6f539e306989de2ff822f8821c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 15:50:48 +0530 Subject: [PATCH 009/100] fix: locking the correct doctype --- .../submission_queue/submission_queue.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 621de6357b..1547177dc7 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -15,6 +15,25 @@ class SubmissionQueue(Document): self.action_for_queuing = action super().insert() + def queue_action(self, action, **kwargs): + from frappe.utils.background_jobs import enqueue + + try: + self.to_be_queued_doc.lock() + except frappe.DocumentLockedError: + frappe.throw( + _("Docuement is already queued for execution"), + title=_("Documenet Queued"), + exc=frappe.DocumentLockedError + ) + return enqueue( + "frappe.model.document.execute_action", + __doctype=self.to_be_queued_doc, + __name=self.to_be_queued_doc.name, + __action=action, + **kwargs + ) + def after_insert(self): job = self.queue_action( "queue_in_background", From b15e07dd791b326cfef81d77616d9c3776747e75 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 15:58:49 +0530 Subject: [PATCH 010/100] refactor: better locking and unlocking --- .../submission_queue/submission_queue.py | 21 ++++--------------- frappe/model/document.py | 1 - 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 1547177dc7..cb3af28a64 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -15,24 +15,11 @@ class SubmissionQueue(Document): self.action_for_queuing = action super().insert() - def queue_action(self, action, **kwargs): - from frappe.utils.background_jobs import enqueue + def lock(self, timeout=None): + self.to_be_queued_doc.lock() - try: - self.to_be_queued_doc.lock() - except frappe.DocumentLockedError: - frappe.throw( - _("Docuement is already queued for execution"), - title=_("Documenet Queued"), - exc=frappe.DocumentLockedError - ) - return enqueue( - "frappe.model.document.execute_action", - __doctype=self.to_be_queued_doc, - __name=self.to_be_queued_doc.name, - __action=action, - **kwargs - ) + def unlock(self): + frappe.get_doc(self.ref_doctype, self.ref_docname).unlock() def after_insert(self): job = self.queue_action( diff --git a/frappe/model/document.py b/frappe/model/document.py index ca27a830d3..aa55eac30a 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -3,7 +3,6 @@ import hashlib import json import time -from datetime import datetime from werkzeug.exceptions import NotFound From d4a0ad436c79b25769ca5c08f7d8a7b6802dc907 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 16:10:25 +0530 Subject: [PATCH 011/100] feat: Added completed_at field and better naming --- .../submission_queue/submission_queue.json | 39 +++++++++++-------- .../submission_queue/submission_queue.py | 7 ++-- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index f2972c5927..2bea4ada4e 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -7,28 +7,17 @@ "engine": "InnoDB", "field_order": [ "status", - "start_time", - "created_by", + "enqueued_at", + "enqueued_by", "job_id", "column_break_5", "ref_doctype", "ref_docname", "section_break_8", - "message" + "message", + "completed_at" ], "fields": [ - { - "fieldname": "start_time", - "fieldtype": "Datetime", - "label": "Start time", - "read_only": 1 - }, - { - "fieldname": "created_by", - "fieldtype": "Data", - "label": "Created By", - "read_only": 1 - }, { "fieldname": "job_id", "fieldtype": "Data", @@ -72,11 +61,29 @@ { "fieldname": "section_break_8", "fieldtype": "Section Break" + }, + { + "fieldname": "enqueued_at", + "fieldtype": "Datetime", + "label": "Enqueued At", + "read_only": 1 + }, + { + "fieldname": "enqueued_by", + "fieldtype": "Data", + "label": "Enqueued By", + "read_only": 1 + }, + { + "fieldname": "completed_at", + "fieldtype": "Data", + "label": "Completed At", + "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-04 18:41:08.495515", + "modified": "2022-10-05 16:09:36.663459", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index cb3af28a64..417657fd77 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -38,7 +38,7 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - values = {"status": "Completed"} + values = {"status": "Completed", "completed_at": datetime.now()} except Exception as e: values = {"status": "Failed", "message": str(e)} @@ -47,6 +47,7 @@ class SubmissionQueue(Document): def notify(name: str): + # Todo: fix notification notification_doc = { "type": "Notification", "document_type": "Submission Queue", @@ -66,8 +67,8 @@ def notify(name: str): def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" - queue.start_time = datetime.now() - queue.created_by = frappe.session.user + queue.enqueued_at = datetime.now() + queue.enqueued_by = frappe.session.user queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) From ae24f17e4265bb7ff2a668d3eadcb45845e271fa Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 22:40:59 +0530 Subject: [PATCH 012/100] refactor: changed field order & added depends on --- frappe/core/doctype/doctype/doctype.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index b43a9ec3ae..c583c959aa 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -23,6 +23,7 @@ "custom", "beta", "is_virtual", + "submit_in_background", "fields_section_break", "fields", "sb1", @@ -58,7 +59,6 @@ "icon", "color", "show_preview_popup", - "submit_in_background", "show_name_in_global_search", "email_settings_sb", "default_email_template", @@ -609,6 +609,7 @@ }, { "default": "0", + "depends_on": "is_submittable", "fieldname": "submit_in_background", "fieldtype": "Check", "label": "Submit in background" @@ -694,7 +695,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-10-03 16:14:46.432770", + "modified": "2022-10-05 22:39:47.594977", "modified_by": "Administrator", "module": "Core", "name": "DocType", From cb796a4e64b49ecbd4ec5e4e03414ea5d3a1ab6d Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 6 Oct 2022 01:07:23 +0530 Subject: [PATCH 013/100] refactor(minor): better notification * changed completed_at fieldname to ended_at (in submission queue doctype) * added rollback on exception in queue_in_background method --- .../submission_queue/submission_queue.json | 12 ++-- .../submission_queue/submission_queue.py | 61 +++++++++++-------- frappe/desk/form/save.py | 10 ++- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 2bea4ada4e..25d59a8d8a 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -11,11 +11,11 @@ "enqueued_by", "job_id", "column_break_5", + "ended_at", "ref_doctype", "ref_docname", "section_break_8", - "message", - "completed_at" + "message" ], "fields": [ { @@ -75,15 +75,15 @@ "read_only": 1 }, { - "fieldname": "completed_at", - "fieldtype": "Data", - "label": "Completed At", + "fieldname": "ended_at", + "fieldtype": "Datetime", + "label": "Ended At", "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-05 16:09:36.663459", + "modified": "2022-10-06 01:11:11.835985", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 417657fd77..705baadc31 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,12 +1,11 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt -from datetime import datetime - import frappe from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document +from frappe.utils import now class SubmissionQueue(Document): @@ -15,10 +14,12 @@ class SubmissionQueue(Document): self.action_for_queuing = action super().insert() - def lock(self, timeout=None): + def lock(self): self.to_be_queued_doc.lock() def unlock(self): + # NOTE: this is called in execute_action method of Document class + # where get_doc is called hence we lose the to_be_queued_doc attribute frappe.get_doc(self.ref_doctype, self.ref_docname).unlock() def after_insert(self): @@ -26,9 +27,14 @@ class SubmissionQueue(Document): "queue_in_background", to_be_queued_doc=self.to_be_queued_doc, action_for_queuing=self.action_for_queuing, + timeout=600, + ) + frappe.db.set_value( + self.doctype, + self.name, + {"job_id": job.id, "enqueued_at": now()}, + update_modified=False, ) - frappe.db.set_value(self.doctype, self.name, {"job_id": job.id}, update_modified=False) - def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() @@ -38,37 +44,44 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - values = {"status": "Completed", "completed_at": datetime.now()} - except Exception as e: - values = {"status": "Failed", "message": str(e)} + values = {"status": "Completed"} + except Exception: + values = {"status": "Failed", "message": frappe.get_traceback()} + frappe.db.rollback() + values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - notify(name=self.name) + self.notify(values["status"], action_for_queuing) + def notify(self, submission_status: str, action: str): + if submission_status == "Failed": + doctype = "Submission Queue" + docname = self.name + message = _("Submission of {0} {1} with action {2} failed") + else: + doctype = self.ref_doctype + docname = self.ref_docname + message = _("Submission of {0} {1} with action {2} completed successfully") -def notify(name: str): - # Todo: fix notification - notification_doc = { - "type": "Notification", - "document_type": "Submission Queue", - "document_name": name, - "subject": "Job Queued", - "from_user": frappe.session.user, - "email_content": "Hello", + notification_doc = { + "type": "Alert", + "document_type": doctype, + "document_name": docname, + "subject": message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ), } - mention = frappe.db.get_value("User", filters=frappe.db.get_value( - "Submission Queue", filters=name, fieldname="created_by" - ), fieldname="email") - if mention: - enqueue_create_notification([mention], notification_doc) + notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") + enqueue_create_notification([notify_to], notification_doc) def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" - queue.enqueued_at = datetime.now() queue.enqueued_by = frappe.session.user queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) + + return queue.name diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index d6a7e70b4d..c633f3610d 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -19,8 +19,14 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - queue_submission(doc, action) - frappe.msgprint(frappe._("Queued for Submission"), indicator="green", alert=True) + queue_name = queue_submission(doc, action) + frappe.msgprint( + frappe._("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) return else: doc.submit() From c0b3928ac154bb08fcb894acd1b9fce130d71949 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 6 Oct 2022 02:03:27 +0530 Subject: [PATCH 014/100] refactor(minor): use virtual docfield for created_at (previously enqueued_at) field --- .../submission_queue/submission_queue.json | 17 +++++++++-------- .../submission_queue/submission_queue.py | 6 +++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 25d59a8d8a..097f233fd4 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -7,7 +7,7 @@ "engine": "InnoDB", "field_order": [ "status", - "enqueued_at", + "created_at", "enqueued_by", "job_id", "column_break_5", @@ -62,12 +62,6 @@ "fieldname": "section_break_8", "fieldtype": "Section Break" }, - { - "fieldname": "enqueued_at", - "fieldtype": "Datetime", - "label": "Enqueued At", - "read_only": 1 - }, { "fieldname": "enqueued_by", "fieldtype": "Data", @@ -79,11 +73,18 @@ "fieldtype": "Datetime", "label": "Ended At", "read_only": 1 + }, + { + "fieldname": "created_at", + "fieldtype": "Datetime", + "is_virtual": 1, + "label": "Created At", + "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-06 01:11:11.835985", + "modified": "2022-10-06 01:57:19.253609", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 705baadc31..3b7f8e10ee 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -9,6 +9,10 @@ from frappe.utils import now class SubmissionQueue(Document): + @property + def created_at(self): + return self.creation + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -32,7 +36,7 @@ class SubmissionQueue(Document): frappe.db.set_value( self.doctype, self.name, - {"job_id": job.id, "enqueued_at": now()}, + {"job_id": job.id}, update_modified=False, ) From bbc8f0baeac33cc576924f7c4d388f6eed0ca997 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 6 Oct 2022 15:22:48 +0530 Subject: [PATCH 015/100] fix: only show submit in background when doc is submittable * chore: better naming --- frappe/core/doctype/doctype/doctype.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index c583c959aa..78d2a43c75 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -23,7 +23,7 @@ "custom", "beta", "is_virtual", - "submit_in_background", + "queue_in_background", "fields_section_break", "fields", "sb1", @@ -609,10 +609,10 @@ }, { "default": "0", - "depends_on": "is_submittable", - "fieldname": "submit_in_background", + "depends_on": "eval: doc.is_submittable", + "fieldname": "queue_in_background", "fieldtype": "Check", - "label": "Submit in background" + "label": "Queue in Background" } ], "icon": "fa fa-bolt", @@ -695,7 +695,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-10-05 22:39:47.594977", + "modified": "2022-10-06 15:20:12.186038", "modified_by": "Administrator", "module": "Core", "name": "DocType", From 48e39de28afbfa528f18839b0a95170a8c5d8867 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 6 Oct 2022 18:21:45 +0530 Subject: [PATCH 016/100] fix(minor): show ref_doctype and ref_docname in listview of submission queue --- frappe/core/doctype/submission_queue/submission_queue.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 097f233fd4..15cb788f18 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -21,13 +21,13 @@ { "fieldname": "job_id", "fieldtype": "Data", - "in_list_view": 1, "label": "Job Id", "read_only": 1 }, { "fieldname": "ref_doctype", "fieldtype": "Link", + "in_list_view": 1, "label": "Reference DocType", "options": "DocType", "read_only": 1 @@ -35,6 +35,7 @@ { "fieldname": "ref_docname", "fieldtype": "Dynamic Link", + "in_list_view": 1, "label": "Reference Docname", "options": "ref_doctype", "read_only": 1 @@ -84,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-06 01:57:19.253609", + "modified": "2022-10-06 18:21:01.129702", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", From c790ad51a398684fb0b083b7da473b23b8a77255 Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 7 Oct 2022 13:14:14 +0530 Subject: [PATCH 017/100] fix(minor): only queue in background for submit action --- .../doctype/submission_queue/submission_queue.py | 8 +++++++- frappe/desk/form/save.py | 16 +++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 3b7f8e10ee..f83f7473e6 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -88,4 +88,10 @@ def queue_submission(doc: Document, action: str): queue.ref_docname = doc.name queue.insert(doc, action) - return queue.name + frappe.msgprint( + frappe._("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index c633f3610d..be41cbfaee 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -7,6 +7,7 @@ import frappe from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.desk.form.load import run_onload from frappe.monitor import add_data_to_monitor +from frappe.utils.scheduler import is_scheduler_inactive @frappe.whitelist() @@ -18,18 +19,11 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if doc.meta.submit_in_background: - queue_name = queue_submission(doc, action) - frappe.msgprint( - frappe._("Queued for Submission. You can track the progress over {0}.").format( - f"here" - ), - indicator="green", - alert=True, - ) + if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): + queue_submission(doc, action) return - else: - doc.submit() + + doc.submit() else: doc.save() From 9074e3e13dc40e4486b50d08ca38677e00544f6f Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 01:36:40 +0530 Subject: [PATCH 018/100] feat: Added unlocked button for locked documents not in queue --- .../doctype/submission_queue/submission_queue.js | 9 ++++++--- .../doctype/submission_queue/submission_queue.py | 15 +++++++++++++++ frappe/model/document.py | 3 +++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index df0833c004..573de63fdb 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -1,7 +1,10 @@ // Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on("Submission Queue", { - // refresh: function(frm) { - // } +frappe.ui.form.on('Submission Queue', { + refresh: function(frm) { + frm.add_custom_button(__("Unlock"), () => { + frm.call("unlock_doc") + }) + } }); diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index f83f7473e6..3d2aae388c 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -6,6 +6,9 @@ from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.utils import now +from rq.exceptions import NoSuchJobError +from rq.job import Job +from frappe.utils.background_jobs import get_redis_conn class SubmissionQueue(Document): @@ -79,6 +82,18 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + @frappe.whitelist() + def unlock_doc(self): + if self.is_locked: + try: + Job(self.job_id, connection=get_redis_conn()) + frappe.msgprint(_("Document already exists in queue!")) + except NoSuchJobError: + self.to_be_queued_doc.unlock() + frappe.msgprint(_("Unlocked document as no such document exists in queue")) + else: + frappe.msgprint(_("Document is already unlocked")) + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") diff --git a/frappe/model/document.py b/frappe/model/document.py index aa55eac30a..f2980a7972 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -91,6 +91,7 @@ class Document(BaseDocument): self.doctype = None self.name = None self.flags = frappe._dict() + self.is_locked = False if args and args[0]: if isinstance(args[0], str): @@ -1492,12 +1493,14 @@ class Document(BaseDocument): raise frappe.DocumentLockedError file_lock.create_lock(signature) frappe.local.locked_documents.append(self) + self.is_locked = True def unlock(self): """Delete the lock file for this document""" file_lock.delete_lock(self.get_signature()) if self in frappe.local.locked_documents: frappe.local.locked_documents.remove(self) + self.is_locked = False # validation helpers def validate_from_to_dates(self, from_date_field, to_date_field): From f28506cafb031e7ae1235f9a2512df7dac811824 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 01:51:27 +0530 Subject: [PATCH 019/100] feat: Added status updates in documents --- frappe/core/doctype/submission_queue/submission_queue.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 3d2aae388c..b1a9aeca53 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -90,11 +90,17 @@ class SubmissionQueue(Document): frappe.msgprint(_("Document already exists in queue!")) except NoSuchJobError: self.to_be_queued_doc.unlock() + self.status = "Failed" + self.save() frappe.msgprint(_("Unlocked document as no such document exists in queue")) else: + # failed, completed don't know at this point + self.status = "Failed" + self.save() frappe.msgprint(_("Document is already unlocked")) + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" From 1000a10c24bd483e79cd4d2c8946662ff22d8ba0 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 03:53:06 +0530 Subject: [PATCH 020/100] fix: fixed job status after fetching job --- .../core/doctype/submission_queue/submission_queue.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b1a9aeca53..ed632ea5d3 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -86,17 +86,13 @@ class SubmissionQueue(Document): def unlock_doc(self): if self.is_locked: try: - Job(self.job_id, connection=get_redis_conn()) - frappe.msgprint(_("Document already exists in queue!")) + job = Job(self.job_id, connection=get_redis_conn()) + if not job.get_status(refresh=True): + raise NoSuchJobError except NoSuchJobError: self.to_be_queued_doc.unlock() - self.status = "Failed" - self.save() frappe.msgprint(_("Unlocked document as no such document exists in queue")) else: - # failed, completed don't know at this point - self.status = "Failed" - self.save() frappe.msgprint(_("Document is already unlocked")) From d8ff47aac2927cc8d3e659e9eb90ec5e475ea8e3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 04:47:37 +0530 Subject: [PATCH 021/100] fix: checking and unlocking the correct doc --- .../submission_queue/submission_queue.js | 14 ++++++++------ .../submission_queue/submission_queue.json | 10 +++++++++- .../submission_queue/submission_queue.py | 18 ++++++++++++------ frappe/desk/form/save.py | 1 - frappe/model/document.py | 2 +- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 573de63fdb..1f652facd8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -1,10 +1,12 @@ // Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Submission Queue', { - refresh: function(frm) { - frm.add_custom_button(__("Unlock"), () => { - frm.call("unlock_doc") - }) - } +frappe.ui.form.on("Submission Queue", { + refresh: function (frm) { + if (frm.doc.status === "Queued") { + frm.add_custom_button(__("Unlock Reference Document"), () => { + frm.call("unlock_doc"); + }); + } + }, }); diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 15cb788f18..c4a4ff4141 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -85,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-06 18:21:01.129702", + "modified": "2022-10-08 04:26:01.657818", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -116,6 +116,14 @@ { "color": "Green", "title": "Completed" + }, + { + "color": "Yellow", + "title": "Stopped" + }, + { + "color": "Red", + "title": "Canceled" } ] } \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index ed632ea5d3..26c284dea8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,13 +1,14 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from rq.exceptions import NoSuchJobError +from rq.job import Job + import frappe from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.utils import now -from rq.exceptions import NoSuchJobError -from rq.job import Job from frappe.utils.background_jobs import get_redis_conn @@ -84,19 +85,24 @@ class SubmissionQueue(Document): @frappe.whitelist() def unlock_doc(self): - if self.is_locked: + to_be_unlocked_doc = frappe.get_doc(self.ref_doctype, self.ref_docname) + if to_be_unlocked_doc.is_locked: try: job = Job(self.job_id, connection=get_redis_conn()) - if not job.get_status(refresh=True): + status = job.get_status(refresh=True) + if not status: raise NoSuchJobError + if status := status in ("failed", "canceled", "stopped"): + to_be_unlocked_doc.unlock() + self.status = status.capitalize() + self.save() except NoSuchJobError: - self.to_be_queued_doc.unlock() + to_be_unlocked_doc.unlock() frappe.msgprint(_("Unlocked document as no such document exists in queue")) else: frappe.msgprint(_("Document is already unlocked")) - def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index be41cbfaee..f43031c899 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -22,7 +22,6 @@ def savedocs(doc, action): if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): queue_submission(doc, action) return - doc.submit() else: doc.save() diff --git a/frappe/model/document.py b/frappe/model/document.py index f2980a7972..8a86ef1bb8 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1498,9 +1498,9 @@ class Document(BaseDocument): def unlock(self): """Delete the lock file for this document""" file_lock.delete_lock(self.get_signature()) + self.is_locked = False if self in frappe.local.locked_documents: frappe.local.locked_documents.remove(self) - self.is_locked = False # validation helpers def validate_from_to_dates(self, from_date_field, to_date_field): From 6099e7e76dcaa52a2ad15e30a06886111aad3d2d Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 17:37:35 +0530 Subject: [PATCH 022/100] feat: Added more statuses for document state --- .../submission_queue/submission_queue.py | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 26c284dea8..812311dcc2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -83,24 +83,37 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + def unlock_doc_and_update_status(self, to_be_unlocked_doc: Document, possible_status: tuple): + try: + if not to_be_unlocked_doc.is_locked: + frappe.msgprint(_("Document is already unlocked updating status")) + + job = Job(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + if not status: + raise NoSuchJobError + + if status == "Queued": + frappe.msgprint(_("Document in queue for execution!")) + return + + if status := status in possible_status: + to_be_unlocked_doc.unlock() + self.status = status + self.save() + frappe.msgprint(_("Document unlocked!")) + + except NoSuchJobError: + to_be_unlocked_doc.unlock() + frappe.msgprint(_("Unlocked document as no such document exists in queue")) + @frappe.whitelist() def unlock_doc(self): + possible_status = ("Failed", "Canceled", "Stopped", "Completed") to_be_unlocked_doc = frappe.get_doc(self.ref_doctype, self.ref_docname) - if to_be_unlocked_doc.is_locked: - try: - job = Job(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - if not status: - raise NoSuchJobError - if status := status in ("failed", "canceled", "stopped"): - to_be_unlocked_doc.unlock() - self.status = status.capitalize() - self.save() - except NoSuchJobError: - to_be_unlocked_doc.unlock() - frappe.msgprint(_("Unlocked document as no such document exists in queue")) - else: - frappe.msgprint(_("Document is already unlocked")) + self.unlock_doc_and_update_status( + to_be_unlocked_doc=to_be_unlocked_doc, possible_status=possible_status + ) def queue_submission(doc: Document, action: str): From eff6c4fc5d64b183a8086bb44b6f08afa4a08f01 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 17:38:12 +0530 Subject: [PATCH 023/100] fix: checking if doc is submittable before queueing --- frappe/desk/form/save.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index f43031c899..949de9e7aa 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -19,7 +19,12 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): + if ( + action == "Submit" + and doc.meta.queue_in_background + and doc.meta.is_submittable + and not is_scheduler_inactive() + ): queue_submission(doc, action) return doc.submit() From 7f21e558dedf3ea34219fce2c6237dbfcc1ac288 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 17:39:07 +0530 Subject: [PATCH 024/100] fix(minor): better naming --- .../doctype/submission_queue/submission_queue.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 812311dcc2..7e3e98839b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -83,9 +83,9 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_doc_and_update_status(self, to_be_unlocked_doc: Document, possible_status: tuple): + def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): try: - if not to_be_unlocked_doc.is_locked: + if not doc_to_be_unlocked.is_locked: frappe.msgprint(_("Document is already unlocked updating status")) job = Job(self.job_id, connection=get_redis_conn()) @@ -98,21 +98,21 @@ class SubmissionQueue(Document): return if status := status in possible_status: - to_be_unlocked_doc.unlock() + doc_to_be_unlocked.unlock() self.status = status self.save() frappe.msgprint(_("Document unlocked!")) except NoSuchJobError: - to_be_unlocked_doc.unlock() + doc_to_be_unlocked.unlock() frappe.msgprint(_("Unlocked document as no such document exists in queue")) @frappe.whitelist() def unlock_doc(self): possible_status = ("Failed", "Canceled", "Stopped", "Completed") - to_be_unlocked_doc = frappe.get_doc(self.ref_doctype, self.ref_docname) + doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) self.unlock_doc_and_update_status( - to_be_unlocked_doc=to_be_unlocked_doc, possible_status=possible_status + doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status ) From 154e3b4e104913a31517319e67ee84d3881e10f3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 23:23:42 +0530 Subject: [PATCH 025/100] refactor: added more states to document when it's queued --- frappe/core/doctype/submission_queue/submission_queue.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index c4a4ff4141..7c744363c2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -46,7 +46,7 @@ "hidden": 1, "in_list_view": 1, "label": "Status", - "options": "Queued\nCompleted\nFailed", + "options": "Queued\nFinished\nFailed\nStopped\nCanceled", "read_only": 1 }, { @@ -85,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-08 04:26:01.657818", + "modified": "2022-10-08 18:07:02.753123", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -115,7 +115,7 @@ }, { "color": "Green", - "title": "Completed" + "title": "Finished" }, { "color": "Yellow", From 9bede49fae64ea960849945ba921aaecc7540cd3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 23:24:07 +0530 Subject: [PATCH 026/100] fix: fixed unlocking of docs from frontend --- .../submission_queue/submission_queue.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 7e3e98839b..b2cf275890 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -52,7 +52,7 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - values = {"status": "Completed"} + values = {"status": "Finished"} except Exception: values = {"status": "Failed", "message": frappe.get_traceback()} frappe.db.rollback() @@ -76,7 +76,9 @@ class SubmissionQueue(Document): "document_type": doctype, "document_name": docname, "subject": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + f"{str(self.ref_doctype)}", + frappe.bold(self.ref_docname), + frappe.bold(action) ), } @@ -85,31 +87,34 @@ class SubmissionQueue(Document): def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): try: - if not doc_to_be_unlocked.is_locked: - frappe.msgprint(_("Document is already unlocked updating status")) - job = Job(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) if not status: raise NoSuchJobError - if status == "Queued": + if not doc_to_be_unlocked.is_locked: + frappe.msgprint(_("Document is already unlocked updating status")) + + # Checking if job is queue to be executed + if status == "queued": frappe.msgprint(_("Document in queue for execution!")) return - if status := status in possible_status: + # Checking any one of the possible termination statuses + if status in possible_status: doc_to_be_unlocked.unlock() - self.status = status + self.status = status.capitalize() self.save() frappe.msgprint(_("Document unlocked!")) except NoSuchJobError: + # Need to update status doc_to_be_unlocked.unlock() frappe.msgprint(_("Unlocked document as no such document exists in queue")) @frappe.whitelist() def unlock_doc(self): - possible_status = ("Failed", "Canceled", "Stopped", "Completed") + possible_status = ("failed", "canceled", "stopped", "finished") doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) self.unlock_doc_and_update_status( doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status From 47c75fd8bb1860f8b2789d9b87307175168b0493 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 17:26:55 +0530 Subject: [PATCH 027/100] refactor: removed redundant checking of locked documents --- frappe/core/doctype/submission_queue/submission_queue.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b2cf275890..49419b5bef 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -76,9 +76,7 @@ class SubmissionQueue(Document): "document_type": doctype, "document_name": docname, "subject": message.format( - f"{str(self.ref_doctype)}", - frappe.bold(self.ref_docname), - frappe.bold(action) + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ), } @@ -92,9 +90,6 @@ class SubmissionQueue(Document): if not status: raise NoSuchJobError - if not doc_to_be_unlocked.is_locked: - frappe.msgprint(_("Document is already unlocked updating status")) - # Checking if job is queue to be executed if status == "queued": frappe.msgprint(_("Document in queue for execution!")) @@ -130,7 +125,7 @@ def queue_submission(doc: Document, action: str): queue.insert(doc, action) frappe.msgprint( - frappe._("Queued for Submission. You can track the progress over {0}.").format( + _("Queued for Submission. You can track the progress over {0}.").format( f"here" ), indicator="green", From b44ad8acb6ff7fae10c0fdd9064f0c8416c26807 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 17:37:02 +0530 Subject: [PATCH 028/100] feat: Adding queue in background to customize form as well --- frappe/custom/doctype/customize_form/customize_form.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 05989eaa00..b22c2aba7a 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -19,6 +19,7 @@ "track_views", "allow_auto_repeat", "allow_import", + "queue_in_background", "fields_section_break", "fields", "naming_section", @@ -337,6 +338,12 @@ "fieldname": "make_attachments_public", "fieldtype": "Check", "label": "Make Attachments Public by Default" + }, + { + "default": "0", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" } ], "hide_toolbar": 1, @@ -345,7 +352,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-08-24 06:57:47.966331", + "modified": "2022-10-09 17:36:03.259470", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", From f45d1b585563b4d6393181c85d6eaf0538093bd7 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 18:25:21 +0530 Subject: [PATCH 029/100] fix: fixed redirection to doctype on notification --- frappe/core/doctype/submission_queue/submission_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 49419b5bef..5c7d8f0616 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -72,7 +72,7 @@ class SubmissionQueue(Document): message = _("Submission of {0} {1} with action {2} completed successfully") notification_doc = { - "type": "Alert", + "type": "Mention", "document_type": doctype, "document_name": docname, "subject": message.format( From 0c00c34ad64d7e20c07caf5cd4b35d3c0cc060e2 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 18:44:28 +0530 Subject: [PATCH 030/100] fix: temporary document locking --- frappe/model/document.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/model/document.py b/frappe/model/document.py index 8a86ef1bb8..63b96ebc5c 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -294,6 +294,10 @@ class Document(BaseDocument): follow_document(self.doctype, self.name, frappe.session.user) return self + def check_locked_document(self): + if self.is_locked: + raise frappe.DocumentLockedError + def save(self, *args, **kwargs): """Wrapper for _save""" return self._save(*args, **kwargs) @@ -310,6 +314,8 @@ class Document(BaseDocument): if self.flags.in_print: return + self.check_locked_document() + self.flags.notifications_executed = [] if ignore_permissions is not None: @@ -988,12 +994,14 @@ class Document(BaseDocument): @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" + self.check_locked_document() self.docstatus = DocStatus.submitted() return self.save() @whitelist.__func__ def _cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves.""" + self.check_locked_document() self.docstatus = DocStatus.cancelled() return self.save() From cde5e634e4f0845ead0bdfa8f631046e2b3e8e8d Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 21:16:59 +0530 Subject: [PATCH 031/100] feat: Adding data to monitor on action --- frappe/core/doctype/submission_queue/submission_queue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 5c7d8f0616..0ddb0945a9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -8,6 +8,7 @@ import frappe from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document +from frappe.monitor import add_data_to_monitor from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn @@ -52,6 +53,7 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() + add_data_to_monitor(doctype=to_be_queued_doc.doctype, action=_action) values = {"status": "Finished"} except Exception: values = {"status": "Failed", "message": frappe.get_traceback()} From b733c82a77237a8e282a6a676f578fd0b4477111 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 21:17:22 +0530 Subject: [PATCH 032/100] feat: Added identifier for locked state of documents refactor(minor): removed is locked setter to avoid redundancy --- frappe/model/document.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 63b96ebc5c..524bbb5fe9 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -91,7 +91,6 @@ class Document(BaseDocument): self.doctype = None self.name = None self.flags = frappe._dict() - self.is_locked = False if args and args[0]: if isinstance(args[0], str): @@ -121,6 +120,10 @@ class Document(BaseDocument): # incorrect arguments. let's not proceed. raise ValueError("Illegal arguments") + @property + def is_locked(self): + return file_lock.lock_exists(self.get_signature()) + @staticmethod def whitelist(fn): """Decorator: Whitelist method to be called remotely via REST API.""" @@ -1412,7 +1415,9 @@ class Document(BaseDocument): def get_signature(self): """Returns signature (hash) for private URL.""" - return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest() + return hashlib.sha224( + get_datetime_str(self.creation or self.modified or now()).encode() + ).hexdigest() def get_document_share_key(self, expires_on=None, no_expiry=False): if no_expiry: @@ -1501,12 +1506,10 @@ class Document(BaseDocument): raise frappe.DocumentLockedError file_lock.create_lock(signature) frappe.local.locked_documents.append(self) - self.is_locked = True def unlock(self): """Delete the lock file for this document""" file_lock.delete_lock(self.get_signature()) - self.is_locked = False if self in frappe.local.locked_documents: frappe.local.locked_documents.remove(self) From 49ca10778cc74aa1878a274dc05b1f79ef5797fc Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 23:21:48 +0530 Subject: [PATCH 033/100] feat: Adding method to clear logs --- frappe/core/doctype/submission_queue/submission_queue.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 0ddb0945a9..7f4c1c396a 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -117,6 +117,14 @@ class SubmissionQueue(Document): doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status ) + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder.functions import Now + from frappe.query_builder import Interval + + table = frappe.qb.DocType("Submission Queue") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") From 563e3c06b0db7d85dc8ec1b94f415ccfc2e1d212 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 17:12:34 +0530 Subject: [PATCH 034/100] refactor: edited unlocked document message --- frappe/core/doctype/submission_queue/submission_queue.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 7f4c1c396a..db95228463 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -86,8 +86,9 @@ class SubmissionQueue(Document): enqueue_create_notification([notify_to], notification_doc) def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): + unlocked_doc_message = "Document Unlocked" try: - job = Job(self.job_id, connection=get_redis_conn()) + job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) if not status: raise NoSuchJobError @@ -102,12 +103,12 @@ class SubmissionQueue(Document): doc_to_be_unlocked.unlock() self.status = status.capitalize() self.save() - frappe.msgprint(_("Document unlocked!")) + frappe.msgprint(_(unlocked_doc_message)) except NoSuchJobError: # Need to update status doc_to_be_unlocked.unlock() - frappe.msgprint(_("Unlocked document as no such document exists in queue")) + frappe.msgprint(_(unlocked_doc_message)) @frappe.whitelist() def unlock_doc(self): From b33efdc6341875db53621a15ab5a5a2dbd93ddea Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 17:16:07 +0530 Subject: [PATCH 035/100] refactor: moved is_submittable validation from save to submission queue --- frappe/core/doctype/submission_queue/submission_queue.py | 8 +++++++- frappe/desk/form/save.py | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index db95228463..40e50b1f4b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -74,7 +74,7 @@ class SubmissionQueue(Document): message = _("Submission of {0} {1} with action {2} completed successfully") notification_doc = { - "type": "Mention", + "type": "Alert", "document_type": doctype, "document_name": docname, "subject": message.format( @@ -128,6 +128,12 @@ class SubmissionQueue(Document): def queue_submission(doc: Document, action: str): + # Allowing only submittable doctypes to be queued + + if not doc.meta.is_submittable: + getattr(doc, action.lower())() + return + queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.enqueued_by = frappe.session.user diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 949de9e7aa..8ea5b120e6 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -22,7 +22,6 @@ def savedocs(doc, action): if ( action == "Submit" and doc.meta.queue_in_background - and doc.meta.is_submittable and not is_scheduler_inactive() ): queue_submission(doc, action) From 01ff3d8bccb441c3cde5cb45652fbc6e9a811ce8 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 20:50:20 +0530 Subject: [PATCH 036/100] fix: fixed locking and checking --- frappe/model/document.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 524bbb5fe9..8e6fe8cf84 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -298,7 +298,7 @@ class Document(BaseDocument): return self def check_locked_document(self): - if self.is_locked: + if self.creation and self.is_locked: raise frappe.DocumentLockedError def save(self, *args, **kwargs): @@ -317,8 +317,6 @@ class Document(BaseDocument): if self.flags.in_print: return - self.check_locked_document() - self.flags.notifications_executed = [] if ignore_permissions is not None: @@ -329,6 +327,7 @@ class Document(BaseDocument): if self.get("__islocal") or not self.get("name"): return self.insert() + self.check_locked_document() self.check_permission("write", "save") self.set_user_and_timestamp() @@ -997,14 +996,12 @@ class Document(BaseDocument): @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" - self.check_locked_document() self.docstatus = DocStatus.submitted() return self.save() @whitelist.__func__ def _cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves.""" - self.check_locked_document() self.docstatus = DocStatus.cancelled() return self.save() @@ -1415,9 +1412,7 @@ class Document(BaseDocument): def get_signature(self): """Returns signature (hash) for private URL.""" - return hashlib.sha224( - get_datetime_str(self.creation or self.modified or now()).encode() - ).hexdigest() + return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest() def get_document_share_key(self, expires_on=None, no_expiry=False): if no_expiry: From 2b7ea8992908299a67d28f0abe933838b8308882 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 21:24:28 +0530 Subject: [PATCH 037/100] feat: Added submittable and queue in background logic --- .../doctype/customize_form/customize_form.json | 13 ++++++++++++- .../custom/doctype/customize_form/customize_form.py | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index b22c2aba7a..0d71aff577 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -12,6 +12,7 @@ "label", "search_fields", "column_break_5", + "is_submittable", "istable", "editable_grid", "quick_entry", @@ -341,9 +342,19 @@ }, { "default": "0", + "depends_on": "eval: doc.is_submittable", "fieldname": "queue_in_background", "fieldtype": "Check", "label": "Queue in Background" + }, + { + "default": "0", + "depends_on": "eval: doc.is_submittable", + "fetch_from": "doc_type.is_submittable", + "fieldname": "is_submittable", + "fieldtype": "Check", + "label": "Is Submittable", + "read_only": 1 } ], "hide_toolbar": 1, @@ -352,7 +363,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-10-09 17:36:03.259470", + "modified": "2022-10-11 21:23:36.669135", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index cacd38397a..2a42d249fc 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -569,6 +569,7 @@ doctype_properties = { "sort_order": "Data", "default_print_format": "Data", "allow_copy": "Check", + "is_submittable": "Check", "istable": "Check", "quick_entry": "Check", "editable_grid": "Check", From a343c06102d7d6cdb9062a03af7784d7da435a26 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 21:25:45 +0530 Subject: [PATCH 038/100] refactor: better naming --- frappe/core/doctype/submission_queue/submission_queue.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 40e50b1f4b..4bc089bd93 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -85,7 +85,7 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): + def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, termination_statues: tuple): unlocked_doc_message = "Document Unlocked" try: job = Job.fetch(self.job_id, connection=get_redis_conn()) @@ -99,7 +99,7 @@ class SubmissionQueue(Document): return # Checking any one of the possible termination statuses - if status in possible_status: + if status in termination_statues: doc_to_be_unlocked.unlock() self.status = status.capitalize() self.save() @@ -112,10 +112,10 @@ class SubmissionQueue(Document): @frappe.whitelist() def unlock_doc(self): - possible_status = ("failed", "canceled", "stopped", "finished") + termination_statues = ("failed", "canceled", "stopped", "finished") doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) self.unlock_doc_and_update_status( - doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status + doc_to_be_unlocked=doc_to_be_unlocked, termination_statues=termination_statues ) @staticmethod From 418f515766aea9876e1fd159124a0fa2a2a24f7b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 21:29:01 +0530 Subject: [PATCH 039/100] feat: Added auto log clearing --- frappe/core/doctype/log_settings/log_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 4a519dcaf4..f1b5d23c4f 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -17,6 +17,7 @@ DEFAULT_LOGTYPES_RETENTION = { "Error Snapshot": 30, "Scheduled Job Log": 90, "Route History": 90, + "Submission Queue": 30, } @@ -151,6 +152,7 @@ LOG_DOCTYPES = [ "Email Queue Recipient", "Error Snapshot", "Error Log", + "Submission Queue", ] From 9681d1965845609ccc8f2631f796245bd9b56de4 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 12 Oct 2022 23:58:19 +0530 Subject: [PATCH 040/100] feat: introduced file for tracking queues --- .../submission_queue/submission_queue.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4bc089bd93..4e46407732 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -11,6 +11,13 @@ from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn +from rq import get_current_job + + +class SubmissionQueueEntries(dict): + def save(self): + with open("./submission_queue_entries.json", "w+") as f: + f.write(frappe.json.dumps(self, indent=4)) class SubmissionQueue(Document): @@ -18,6 +25,10 @@ class SubmissionQueue(Document): def created_at(self): return self.creation + def __init__(self, *args, **kwargs): + self.queue_entries = read_submission_queue_entries() + super().__init__(*args, **kwargs) + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -38,6 +49,13 @@ class SubmissionQueue(Document): action_for_queuing=self.action_for_queuing, timeout=600, ) + self.queue_entries[job.id] = { + "DocType": self.ref_doctype, + "Docname": self.ref_docname, + "Status": job.get_status(refresh=True) + } + self.queue_entries.save() + frappe.db.set_value( self.doctype, self.name, @@ -47,6 +65,7 @@ class SubmissionQueue(Document): def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() + job = get_current_job(connection=get_redis_conn()) if _action == "update": _action = "submit" @@ -61,6 +80,13 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + self.queue_entries[job.id] = { + "DocType": to_be_queued_doc.doctype, + "Docname": to_be_queued_doc.name, + "Status": values["status"] + } + self.queue_entries.save() + self.notify(values["status"], action_for_queuing) def notify(self, submission_status: str, action: str): @@ -86,6 +112,9 @@ class SubmissionQueue(Document): enqueue_create_notification([notify_to], notification_doc) def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, termination_statues: tuple): + # Problem: If someone tries to unlock a previously failed job, + # however someone else has already queued that document again + # this will cause the queued document to be unlocked. unlocked_doc_message = "Document Unlocked" try: job = Job.fetch(self.job_id, connection=get_redis_conn()) @@ -120,13 +149,23 @@ class SubmissionQueue(Document): @staticmethod def clear_old_logs(days=30): - from frappe.query_builder.functions import Now from frappe.query_builder import Interval + from frappe.query_builder.functions import Now table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) +def read_submission_queue_entries(): + try: + with open("./submission_queue_entries.json") as f: + return SubmissionQueueEntries(frappe.json.loads(f.read())) + except FileNotFoundError: + with open("./submission_queue_entries.json", "w+") as f: + f.write(frappe.json.dumps({})) + return SubmissionQueueEntries() + + def queue_submission(doc: Document, action: str): # Allowing only submittable doctypes to be queued From 3cae3d057cf7cd387a300b2d10681221d3c1e8ac Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 13 Oct 2022 14:07:11 +0530 Subject: [PATCH 041/100] refactor(minor): made unlock_doc_and_update_status into a simple function * renamed unlock_doc_and_update_status -> unlock_reference_doc * added queued_doc property * renamed check_locked_document -> check_if_locked * reduced the statuses in submission queue * refactored unlock_reference_doc a bit --- .../submission_queue/submission_queue.json | 4 +- .../submission_queue/submission_queue.py | 84 +++++++++---------- frappe/model/document.py | 4 +- 3 files changed, 44 insertions(+), 48 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 7c744363c2..42f26a3dd2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -46,7 +46,7 @@ "hidden": 1, "in_list_view": 1, "label": "Status", - "options": "Queued\nFinished\nFailed\nStopped\nCanceled", + "options": "Queued\nFinished\nFailed", "read_only": 1 }, { @@ -85,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-08 18:07:02.753123", + "modified": "2022-10-13 18:07:02.753123", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4e46407732..41f34bb71b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,6 +1,7 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from rq import get_current_job from rq.exceptions import NoSuchJobError from rq.job import Job @@ -11,7 +12,6 @@ from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn -from rq import get_current_job class SubmissionQueueEntries(dict): @@ -25,6 +25,10 @@ class SubmissionQueue(Document): def created_at(self): return self.creation + @property + def queued_doc(self): + return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + def __init__(self, *args, **kwargs): self.queue_entries = read_submission_queue_entries() super().__init__(*args, **kwargs) @@ -35,24 +39,22 @@ class SubmissionQueue(Document): super().insert() def lock(self): - self.to_be_queued_doc.lock() + self.queued_doc.lock() def unlock(self): - # NOTE: this is called in execute_action method of Document class - # where get_doc is called hence we lose the to_be_queued_doc attribute - frappe.get_doc(self.ref_doctype, self.ref_docname).unlock() + self.queued_doc.unlock() def after_insert(self): job = self.queue_action( - "queue_in_background", - to_be_queued_doc=self.to_be_queued_doc, + "queue", + to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, timeout=600, ) self.queue_entries[job.id] = { "DocType": self.ref_doctype, "Docname": self.ref_docname, - "Status": job.get_status(refresh=True) + "Status": job.get_status(refresh=True), } self.queue_entries.save() @@ -63,7 +65,7 @@ class SubmissionQueue(Document): update_modified=False, ) - def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): + def queue(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() job = get_current_job(connection=get_redis_conn()) @@ -83,7 +85,7 @@ class SubmissionQueue(Document): self.queue_entries[job.id] = { "DocType": to_be_queued_doc.doctype, "Docname": to_be_queued_doc.name, - "Status": values["status"] + "Status": values["status"], } self.queue_entries.save() @@ -91,7 +93,7 @@ class SubmissionQueue(Document): def notify(self, submission_status: str, action: str): if submission_status == "Failed": - doctype = "Submission Queue" + doctype = self.doctype docname = self.name message = _("Submission of {0} {1} with action {2} failed") else: @@ -111,41 +113,12 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, termination_statues: tuple): - # Problem: If someone tries to unlock a previously failed job, - # however someone else has already queued that document again - # this will cause the queued document to be unlocked. - unlocked_doc_message = "Document Unlocked" - try: - job = Job.fetch(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - if not status: - raise NoSuchJobError - - # Checking if job is queue to be executed - if status == "queued": - frappe.msgprint(_("Document in queue for execution!")) - return - - # Checking any one of the possible termination statuses - if status in termination_statues: - doc_to_be_unlocked.unlock() - self.status = status.capitalize() - self.save() - frappe.msgprint(_(unlocked_doc_message)) - - except NoSuchJobError: - # Need to update status - doc_to_be_unlocked.unlock() - frappe.msgprint(_(unlocked_doc_message)) - @frappe.whitelist() def unlock_doc(self): - termination_statues = ("failed", "canceled", "stopped", "finished") - doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) - self.unlock_doc_and_update_status( - doc_to_be_unlocked=doc_to_be_unlocked, termination_statues=termination_statues - ) + if self.status != "Queued": + return + + unlock_reference_doc(self.queued_doc, self.job_id, self.name) @staticmethod def clear_old_logs(days=30): @@ -166,6 +139,29 @@ def read_submission_queue_entries(): return SubmissionQueueEntries() +def unlock_reference_doc(ref_doc: Document, job_id: str, submission_name: str): + # TODO: If someone tries to unlock a previously failed job, + # and someone else has already queued that document again + # this will cause the queued document to be unlocked. + + try: + job = Job.fetch(job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + except NoSuchJobError: + # assuming the job failed here (?) + status = "failed" + + # Checking if job is queue to be executed/executing + if status in ("queued", "started"): + frappe.msgprint(_("Document in queue for execution!")) + + # Checking any one of the possible termination statuses + elif status in ("failed", "canceled", "stopped"): + ref_doc.unlock() + frappe.db.set_value("Submission Queue", submission_name, "status", "Failed") + frappe.msgprint(_("Document Unlocked")) + + def queue_submission(doc: Document, action: str): # Allowing only submittable doctypes to be queued diff --git a/frappe/model/document.py b/frappe/model/document.py index 8d503e5235..99e51765af 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -299,7 +299,7 @@ class Document(BaseDocument): follow_document(self.doctype, self.name, frappe.session.user) return self - def check_locked_document(self): + def check_if_locked(self): if self.creation and self.is_locked: raise frappe.DocumentLockedError @@ -329,7 +329,7 @@ class Document(BaseDocument): if self.get("__islocal") or not self.get("name"): return self.insert() - self.check_locked_document() + self.check_if_locked() self.check_permission("write", "save") self.set_user_and_timestamp() From 2547cd2d51fd6ef103b3fe28a8c79d5409699dc3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 13 Oct 2022 15:26:40 +0530 Subject: [PATCH 042/100] feat: realtime alerts if user is logged in --- .../submission_queue/submission_queue.js | 10 ++++++ .../submission_queue/submission_queue.py | 31 +++++++++++++------ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 1f652facd8..4ec0ccfde3 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -2,6 +2,16 @@ // For license information, please see license.txt frappe.ui.form.on("Submission Queue", { + setup: frappe.realtime.on("termination_status", (data) => { + let color = "green"; + if (data.status == "Failed") { + color = "orange"; + } + frappe.show_alert({ + message: data.message, + indicator: color, + }); + }), refresh: function (frm) { if (frm.doc.status === "Queued") { frm.add_custom_button(__("Unlock Reference Document"), () => { diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 41f34bb71b..bc95506f26 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -101,17 +101,28 @@ class SubmissionQueue(Document): docname = self.ref_docname message = _("Submission of {0} {1} with action {2} completed successfully") - notification_doc = { - "type": "Alert", - "document_type": doctype, - "document_name": docname, - "subject": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ), - } + if self.enqueued_by == frappe.session.user: + frappe.publish_realtime( + "termination_status", + { + "message": message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ), + "status": submission_status, + }, + ) + else: + notification_doc = { + "type": "Alert", + "document_type": doctype, + "document_name": docname, + "subject": message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ), + } - notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") - enqueue_create_notification([notify_to], notification_doc) + notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") + enqueue_create_notification([notify_to], notification_doc) @frappe.whitelist() def unlock_doc(self): From f4683cc718bc07947afbede72c0da80766db2228 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 13 Oct 2022 16:12:56 +0530 Subject: [PATCH 043/100] chore: remove unnecessary condition --- .../core/doctype/submission_queue/submission_queue.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index bc95506f26..b6b17b7cee 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -169,17 +169,13 @@ def unlock_reference_doc(ref_doc: Document, job_id: str, submission_name: str): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): ref_doc.unlock() - frappe.db.set_value("Submission Queue", submission_name, "status", "Failed") + frappe.db.set_value( + "Submission Queue", submission_name, "status", "Failed", update_modified=False + ) frappe.msgprint(_("Document Unlocked")) def queue_submission(doc: Document, action: str): - # Allowing only submittable doctypes to be queued - - if not doc.meta.is_submittable: - getattr(doc, action.lower())() - return - queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.enqueued_by = frappe.session.user From 6ea2e226f2055f595b6dbe0def096457954e33b0 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 13 Oct 2022 16:30:06 +0530 Subject: [PATCH 044/100] feat: Redirecting to doctype on alert --- frappe/core/doctype/submission_queue/submission_queue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index bc95506f26..c5f681b997 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -107,7 +107,8 @@ class SubmissionQueue(Document): { "message": message.format( frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ), + ) + + f" view it here", "status": submission_status, }, ) From 6a1f8645c3e1fbde9adbc4f25a6fc3bb318e0962 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 13 Oct 2022 17:11:49 +0530 Subject: [PATCH 045/100] feat: clearning submission queue entries & additional check for unlocking documents --- .../submission_queue/submission_queue.py | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index faa0e6bc36..5dedb6d794 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -125,12 +125,40 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + def unlock_reference_doc(self): + try: + job = Job.fetch(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + except NoSuchJobError: + # assuming the job failed here (?) + status = "failed" + + for docs in self.queue_entries.values(): + if ( + docs["DocType"] == self.ref_doctype + and docs["Docname"] == self.ref_docname + and docs["Status"] == "Queued" + ): + status = "queued" + break + # Checking if job is queue to be executed/executing + if status in ("queued", "started"): + frappe.msgprint(_("Document in queue for execution!")) + + # Checking any one of the possible termination statuses + elif status in ("failed", "canceled", "stopped"): + self.queued_doc.unlock() + frappe.db.set_value( + "Submission Queue", self.name, "status", "Failed", update_modified=False + ) + frappe.msgprint(_("Document Unlocked")) + @frappe.whitelist() def unlock_doc(self): if self.status != "Queued": return - unlock_reference_doc(self.queued_doc, self.job_id, self.name) + self.unlock_reference_doc() @staticmethod def clear_old_logs(days=30): @@ -139,6 +167,7 @@ class SubmissionQueue(Document): table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + SubmissionQueueEntries().save() def read_submission_queue_entries(): @@ -151,31 +180,6 @@ def read_submission_queue_entries(): return SubmissionQueueEntries() -def unlock_reference_doc(ref_doc: Document, job_id: str, submission_name: str): - # TODO: If someone tries to unlock a previously failed job, - # and someone else has already queued that document again - # this will cause the queued document to be unlocked. - - try: - job = Job.fetch(job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - except NoSuchJobError: - # assuming the job failed here (?) - status = "failed" - - # Checking if job is queue to be executed/executing - if status in ("queued", "started"): - frappe.msgprint(_("Document in queue for execution!")) - - # Checking any one of the possible termination statuses - elif status in ("failed", "canceled", "stopped"): - ref_doc.unlock() - frappe.db.set_value( - "Submission Queue", submission_name, "status", "Failed", update_modified=False - ) - frappe.msgprint(_("Document Unlocked")) - - def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" From 2083f9f17b4014a450c02da31044a32c74bf6561 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 14 Oct 2022 18:00:32 +0530 Subject: [PATCH 046/100] refactor: removed state management via json file for queues --- .../submission_queue/submission_queue.py | 44 +------------------ 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 5dedb6d794..1bd730a727 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -14,12 +14,6 @@ from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn -class SubmissionQueueEntries(dict): - def save(self): - with open("./submission_queue_entries.json", "w+") as f: - f.write(frappe.json.dumps(self, indent=4)) - - class SubmissionQueue(Document): @property def created_at(self): @@ -30,7 +24,6 @@ class SubmissionQueue(Document): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) def __init__(self, *args, **kwargs): - self.queue_entries = read_submission_queue_entries() super().__init__(*args, **kwargs) def insert(self, to_be_queued_doc: Document, action: str): @@ -51,13 +44,6 @@ class SubmissionQueue(Document): action_for_queuing=self.action_for_queuing, timeout=600, ) - self.queue_entries[job.id] = { - "DocType": self.ref_doctype, - "Docname": self.ref_docname, - "Status": job.get_status(refresh=True), - } - self.queue_entries.save() - frappe.db.set_value( self.doctype, self.name, @@ -82,13 +68,6 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - self.queue_entries[job.id] = { - "DocType": to_be_queued_doc.doctype, - "Docname": to_be_queued_doc.name, - "Status": values["status"], - } - self.queue_entries.save() - self.notify(values["status"], action_for_queuing) def notify(self, submission_status: str, action: str): @@ -133,14 +112,6 @@ class SubmissionQueue(Document): # assuming the job failed here (?) status = "failed" - for docs in self.queue_entries.values(): - if ( - docs["DocType"] == self.ref_doctype - and docs["Docname"] == self.ref_docname - and docs["Status"] == "Queued" - ): - status = "queued" - break # Checking if job is queue to be executed/executing if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) @@ -148,9 +119,7 @@ class SubmissionQueue(Document): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): self.queued_doc.unlock() - frappe.db.set_value( - "Submission Queue", self.name, "status", "Failed", update_modified=False - ) + frappe.db.set_value("Submission Queue", self.name, "status", "Failed", update_modified=False) frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() @@ -167,17 +136,6 @@ class SubmissionQueue(Document): table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) - SubmissionQueueEntries().save() - - -def read_submission_queue_entries(): - try: - with open("./submission_queue_entries.json") as f: - return SubmissionQueueEntries(frappe.json.loads(f.read())) - except FileNotFoundError: - with open("./submission_queue_entries.json", "w+") as f: - f.write(frappe.json.dumps({})) - return SubmissionQueueEntries() def queue_submission(doc: Document, action: str): From eebbccbdbe28bed1b3dfac1d1edc4fbe46b19bda Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 14 Oct 2022 18:03:14 +0530 Subject: [PATCH 047/100] fix: added ignore permissions flag for doctype creations by other users --- frappe/core/doctype/submission_queue/submission_queue.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 1bd730a727..8870b64dc9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -23,13 +23,10 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action - super().insert() + super().insert(ignore_permissions=True) def lock(self): self.queued_doc.lock() From ae3c547e9e51cd8939089ac88970988b2fee0790 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 00:52:16 +0530 Subject: [PATCH 048/100] test: checking persistant locks on document instances --- frappe/tests/test_document_locks.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/frappe/tests/test_document_locks.py b/frappe/tests/test_document_locks.py index a92c9ffc54..2ba606685e 100644 --- a/frappe/tests/test_document_locks.py +++ b/frappe/tests/test_document_locks.py @@ -16,3 +16,24 @@ class TestDocumentLocks(FrappeTestCase): todo_1.lock() self.assertRaises(frappe.DocumentLockedError, todo.lock) todo_1.unlock() + + def test_operations_on_locked_documents(self): + todo = frappe.get_doc(dict(doctype="ToDo", description="testing operations")).insert() + todo.lock() + + with self.assertRaises(frappe.DocumentLockedError): + todo.description = "Random" + todo.save() + + # Checking for persistant locks across all instances. + doc = frappe.get_doc("ToDo", todo.name) + self.assertEquals(doc.is_locked, True) + + with self.assertRaises(frappe.DocumentLockedError): + doc.description = "Random" + doc.save() + + doc.unlock() + self.assertEquals(doc.is_locked, False) + self.assertEquals(todo.is_locked, False) + From 8bb171932d4926c1a2dca0b925fe275f92acc2ca Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 11:57:48 +0530 Subject: [PATCH 049/100] refactor: removed unnecesary actions from queue --- frappe/core/doctype/submission_queue/submission_queue.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 8870b64dc9..e0e7f8f017 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,7 +1,6 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt -from rq import get_current_job from rq.exceptions import NoSuchJobError from rq.job import Job @@ -50,7 +49,6 @@ class SubmissionQueue(Document): def queue(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() - job = get_current_job(connection=get_redis_conn()) if _action == "update": _action = "submit" From 3a10cf4977fe4ec97f276cb43768e7738c9436b9 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 13:39:32 +0530 Subject: [PATCH 050/100] fix: fixed realtime updates --- .../submission_queue/submission_queue.js | 10 --------- .../submission_queue/submission_queue.py | 22 +++++++++++++------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 4ec0ccfde3..1f652facd8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -2,16 +2,6 @@ // For license information, please see license.txt frappe.ui.form.on("Submission Queue", { - setup: frappe.realtime.on("termination_status", (data) => { - let color = "green"; - if (data.status == "Failed") { - color = "orange"; - } - frappe.show_alert({ - message: data.message, - indicator: color, - }); - }), refresh: function (frm) { if (frm.doc.status === "Queued") { frm.add_custom_button(__("Unlock Reference Document"), () => { diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index e0e7f8f017..6c40cf4814 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -22,6 +22,10 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + @property + def enqueued_by(self): + return self.owner + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -75,15 +79,20 @@ class SubmissionQueue(Document): docname = self.ref_docname message = _("Submission of {0} {1} with action {2} completed successfully") + message = ( + message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ) + + f" view it here" + ) + if self.enqueued_by == frappe.session.user: frappe.publish_realtime( - "termination_status", + "msgprint", { - "message": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ) - + f" view it here", - "status": submission_status, + "message": message, + "alert": True, + "indicator": "orange" if submission_status == "Failed" else "green", }, ) else: @@ -136,7 +145,6 @@ class SubmissionQueue(Document): def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" - queue.enqueued_by = frappe.session.user queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) From 3852eaea74b89dc4a0a92fe06a85550044f8aeac Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 14:41:08 +0530 Subject: [PATCH 051/100] feat: Added on_success and on_failure to enqueue --- frappe/utils/background_jobs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index e88cd75efb..98419c321f 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -53,6 +53,8 @@ def enqueue( method, queue="default", timeout=None, + on_success=None, + on_failure=None, event=None, is_async=True, job_name=None, @@ -117,6 +119,8 @@ def enqueue( return q.enqueue_call( execute_job, + on_success=on_success, + on_failure=on_failure, timeout=timeout, kwargs=queue_args, at_front=at_front, From d39c917284da1196464332a26fccc8ad3b17bd8c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 18:47:51 +0530 Subject: [PATCH 052/100] test: Added test for asserting queue action --- .../doctype/submission_queue/submission_queue.py | 4 ++++ .../submission_queue/test_submission_queue.py | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 6c40cf4814..50622b1b91 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -116,6 +116,10 @@ class SubmissionQueue(Document): # assuming the job failed here (?) status = "failed" + # Job finished successfully however action was never completed (?) + if status == "finished" and self.queued_doc.docstatus != 1: + status = "failed" + # Checking if job is queue to be executed/executing if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index d7547983a2..a9cdebdcff 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,9 +1,19 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt -# import frappe +import frappe from frappe.tests.utils import FrappeTestCase +from frappe.utils.background_jobs import get_queue class TestSubmissionQueue(FrappeTestCase): - pass + queue = get_queue(qtype="default") + + def test_queue_creation(self): + from frappe.core.doctype.submission_queue.submission_queue import queue_submission + + doc = frappe.get_doc({"doctype": "ToDo", "description": "Something"}).insert() + queue_submission(doc, "submit") + submission_queue = frappe.get_last_doc("Submission Queue") + job = self.queue.fetch_job(submission_queue.job_id) + self.assertEqual(job.get_status(refresh=True), "queued") From 0eb2458278f3bb55219648ff7e537706503ad284 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 21:49:33 +0530 Subject: [PATCH 053/100] test: fixed queueing test and added completion check --- .../submission_queue/test_submission_queue.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index a9cdebdcff..ad19b3e592 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,6 +1,8 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt +import time + import frappe from frappe.tests.utils import FrappeTestCase from frappe.utils.background_jobs import get_queue @@ -9,11 +11,18 @@ from frappe.utils.background_jobs import get_queue class TestSubmissionQueue(FrappeTestCase): queue = get_queue(qtype="default") - def test_queue_creation(self): + def test_queue_operation(self): from frappe.core.doctype.submission_queue.submission_queue import queue_submission doc = frappe.get_doc({"doctype": "ToDo", "description": "Something"}).insert() queue_submission(doc, "submit") submission_queue = frappe.get_last_doc("Submission Queue") + + # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) - self.assertEqual(job.get_status(refresh=True), "queued") + self.assertIn(job.get_status(refresh=True), ("queued", "started")) + + time.sleep(2) + + # Test completion + self.assertEqual(job.get_status(refresh=True), "finished") From 2172d5f40605128f6bcf253bbb3ddb1f8cdadf90 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 16 Oct 2022 18:32:36 +0530 Subject: [PATCH 054/100] test: removed completion check --- .../core/doctype/submission_queue/test_submission_queue.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index ad19b3e592..3ec99f06b2 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,8 +1,6 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt -import time - import frappe from frappe.tests.utils import FrappeTestCase from frappe.utils.background_jobs import get_queue @@ -21,8 +19,3 @@ class TestSubmissionQueue(FrappeTestCase): # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) self.assertIn(job.get_status(refresh=True), ("queued", "started")) - - time.sleep(2) - - # Test completion - self.assertEqual(job.get_status(refresh=True), "finished") From ff1389327a424ca8940dcde4951ef78a821f5bbd Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 18:23:20 +0530 Subject: [PATCH 055/100] chore: rename queue to background_submission --- .../doctype/submission_queue/submission_queue.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 50622b1b91..9b4411caae 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -18,14 +18,14 @@ class SubmissionQueue(Document): def created_at(self): return self.creation - @property - def queued_doc(self): - return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) - @property def enqueued_by(self): return self.owner + @property + def queued_doc(self): + return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -39,7 +39,7 @@ class SubmissionQueue(Document): def after_insert(self): job = self.queue_action( - "queue", + "background_submission", to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, timeout=600, @@ -51,7 +51,7 @@ class SubmissionQueue(Document): update_modified=False, ) - def queue(self, to_be_queued_doc: Document, action_for_queuing: str): + def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() if _action == "update": @@ -86,6 +86,7 @@ class SubmissionQueue(Document): + f" view it here" ) + # TODO: this is messed up if self.enqueued_by == frappe.session.user: frappe.publish_realtime( "msgprint", From 1856d2eb3f42cab4d5d88f176d2b56349092e819 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 19:11:13 +0530 Subject: [PATCH 056/100] fix: realtime alert use session object's last updated time to figure out whether to send a realtime alert or a notification --- .../submission_queue/submission_queue.py | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 9b4411caae..69523ad3cd 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,6 +1,8 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from datetime import datetime + from rq.exceptions import NoSuchJobError from rq.job import Job @@ -9,7 +11,7 @@ from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.monitor import add_data_to_monitor -from frappe.utils import now +from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds from frappe.utils.background_jobs import get_redis_conn @@ -26,9 +28,10 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) - def insert(self, to_be_queued_doc: Document, action: str): + def insert(self, to_be_queued_doc: Document, action: str, session_id: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action + self.enqueued_by_session_id = session_id super().insert(ignore_permissions=True) def lock(self): @@ -42,6 +45,7 @@ class SubmissionQueue(Document): "background_submission", to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, + enqueued_by_session_id=self.enqueued_by_session_id, timeout=600, ) frappe.db.set_value( @@ -51,7 +55,9 @@ class SubmissionQueue(Document): update_modified=False, ) - def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): + def background_submission( + self, to_be_queued_doc: Document, action_for_queuing: str, enqueued_by_session_id: str + ): _action = action_for_queuing.lower() if _action == "update": @@ -67,9 +73,9 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - self.notify(values["status"], action_for_queuing) + self.notify(values["status"], action_for_queuing, enqueued_by_session_id) - def notify(self, submission_status: str, action: str): + def notify(self, submission_status: str, action: str, session_id: str): if submission_status == "Failed": doctype = self.doctype docname = self.name @@ -79,37 +85,33 @@ class SubmissionQueue(Document): docname = self.ref_docname message = _("Submission of {0} {1} with action {2} completed successfully") - message = ( - message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ) - + f" view it here" + message = message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - # TODO: this is messed up - if self.enqueued_by == frappe.session.user: + if time_diff_in_seconds(now(), get_user_last_request_time(session_id)) < 60: frappe.publish_realtime( "msgprint", { - "message": message, + "message": message + + f". View it here", "alert": True, - "indicator": "orange" if submission_status == "Failed" else "green", + "indicator": "red" if submission_status == "Failed" else "green", }, + user=self.enqueued_by, ) else: notification_doc = { "type": "Alert", "document_type": doctype, "document_name": docname, - "subject": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ), + "subject": message, } notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_reference_doc(self): + def _unlock_reference_doc(self): try: job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) @@ -136,7 +138,7 @@ class SubmissionQueue(Document): if self.status != "Queued": return - self.unlock_reference_doc() + self._unlock_reference_doc() @staticmethod def clear_old_logs(days=30): @@ -147,12 +149,21 @@ class SubmissionQueue(Document): frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) +def get_user_last_request_time(session_id): + return ( + frappe.cache() + .hget("session", session_id) + .get("data", {}) + .get("last_updated", datetime.min.strftime(DATETIME_FORMAT)) + ) + + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.ref_doctype = doc.doctype queue.ref_docname = doc.name - queue.insert(doc, action) + queue.insert(doc, action, frappe.session.sid) frappe.msgprint( _("Queued for Submission. You can track the progress over {0}.").format( From a21eb1efd6648a838d5dc6b0aff28b2c14de12a2 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 19:19:32 +0530 Subject: [PATCH 057/100] refactor(minor): move staticmethod after properties --- .../submission_queue/submission_queue.json | 3 ++- .../doctype/submission_queue/submission_queue.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 42f26a3dd2..d0579ac599 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -66,6 +66,7 @@ { "fieldname": "enqueued_by", "fieldtype": "Data", + "is_virtual": 1, "label": "Enqueued By", "read_only": 1 }, @@ -85,7 +86,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-13 18:07:02.753123", + "modified": "2022-10-30 19:16:59.284681", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 69523ad3cd..34ccfcb482 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -28,6 +28,14 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Submission Queue") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + def insert(self, to_be_queued_doc: Document, action: str, session_id: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -140,14 +148,6 @@ class SubmissionQueue(Document): self._unlock_reference_doc() - @staticmethod - def clear_old_logs(days=30): - from frappe.query_builder import Interval - from frappe.query_builder.functions import Now - - table = frappe.qb.DocType("Submission Queue") - frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) - def get_user_last_request_time(session_id): return ( From 2952077a6d54bda3832f69a662027ddc5329ea8b Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 19:23:15 +0530 Subject: [PATCH 058/100] refactor(minor): remove additional document states and rename message to exception --- .../submission_queue/submission_queue.json | 24 +++++++------------ .../submission_queue/submission_queue.py | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index d0579ac599..00e841b926 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -15,7 +15,7 @@ "ref_doctype", "ref_docname", "section_break_8", - "message" + "exception" ], "fields": [ { @@ -53,12 +53,6 @@ "fieldname": "column_break_5", "fieldtype": "Column Break" }, - { - "fieldname": "message", - "fieldtype": "Text", - "label": "Message", - "read_only": 1 - }, { "fieldname": "section_break_8", "fieldtype": "Section Break" @@ -82,11 +76,17 @@ "is_virtual": 1, "label": "Created At", "read_only": 1 + }, + { + "fieldname": "exception", + "fieldtype": "Text", + "label": "Exception", + "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-30 19:16:59.284681", + "modified": "2022-10-30 19:22:20.998753", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -117,14 +117,6 @@ { "color": "Green", "title": "Finished" - }, - { - "color": "Yellow", - "title": "Stopped" - }, - { - "color": "Red", - "title": "Canceled" } ] } \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 34ccfcb482..e7269b5d37 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -76,7 +76,7 @@ class SubmissionQueue(Document): add_data_to_monitor(doctype=to_be_queued_doc.doctype, action=_action) values = {"status": "Finished"} except Exception: - values = {"status": "Failed", "message": frappe.get_traceback()} + values = {"status": "Failed", "exception": frappe.get_traceback()} frappe.db.rollback() values["ended_at"] = now() From 759ee1a1164901abc329aed10f28d11e89a2500c Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 23:16:09 +0530 Subject: [PATCH 059/100] feat(minor): submission queue banner for showing recent submissions --- .../submission_queue/submission_queue.py | 13 ++++ frappe/public/js/frappe/form/form.js | 63 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index e7269b5d37..37d07f3239 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -172,3 +172,16 @@ def queue_submission(doc: Document, action: str): indicator="green", alert=True, ) + + +@frappe.whitelist() +def get_latest_submissions(doctype, docname): + # NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere + # hence assuming modified will be equal to creation for submission queue documents + + dt = "Submission Queue" + filters = {"ref_doctype": doctype, "ref_docname": docname} + return { + "latest_submission": frappe.db.get_value(dt, filters), + "latest_failed_submission": frappe.db.get_value(dt, filters.update({"status": "Failed"})), + } diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8c642a73f0..6999edbea2 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -450,6 +450,7 @@ frappe.ui.form.Form = class FrappeForm { .toggleClass("cancelled-form", this.doc.docstatus === 2); this.show_conflict_message(); + this.show_submission_queue_banner(); if (frappe.boot.read_only) { this.disable_form(); @@ -2036,6 +2037,68 @@ frappe.ui.form.Form = class FrappeForm { .filter((user) => !["Administrator", frappe.session.user].includes(user)) .filter(Boolean); } + + show_submission_queue_banner() { + if ( + !( + this.meta.is_submittable && + this.meta.queue_in_background && + !this.doc.__islocal && + this.doc.docstatus === 0 + ) + ) + return; + + let wrapper = this.layout.wrapper.find(".submission-queue-banner"); + if (!wrapper.length) { + wrapper = $('
'); + this.layout.wrapper.prepend(wrapper); + } + + frappe + .call({ + method: "frappe.core.doctype.submission_queue.submission_queue.get_latest_submissions", + args: { doctype: this.doctype, docname: this.docname }, + }) + .then((r) => { + if (r.message.latest_submission) { + // if we are here that means some submission(s) were queued and are in queued/failed state + wrapper.show(); + let col_width = 4; + let failed_link = ""; + if (r.message.latest_failed_submission) { + col_width = 3; + failed_link = `
+ ${__( + "Latest Failed Submission" + )} +
`; + } + + let html = ` +
+
+ ${__("Submission Status:")} +
+
+ ${__("Latest Submission")} +
+ ${failed_link} +
+ ${__( + "All Submissions" + )} +
+
+ `; + + wrapper.html(html); + } else { + wrapper.hide(); + wrapper.html(""); + } + }); + } }; frappe.validated = 0; From c3e2cd1629aeab25fdc552ebf229d942698d603c Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 23:42:14 +0530 Subject: [PATCH 060/100] fix: only allow queue_in_background in customize form --- .../doctype/customize_form/customize_form.js | 4 ++++ .../customize_form/customize_form.json | 23 +++++-------------- .../doctype/customize_form/customize_form.py | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 759a9e1b3a..6f94f32256 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -149,6 +149,10 @@ frappe.ui.form.on("Customize Form", { const is_autoname_autoincrement = frm.doc.autoname === "autoincrement"; frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); + frm.toggle_display( + ["queue_in_background"], + frappe.get_meta(frm.doc.doc_type).is_submittable || 0 + ); } frm.events.setup_export(frm); diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index a64d1fa05a..b9fb52d1dc 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -12,7 +12,6 @@ "label", "search_fields", "column_break_5", - "is_submittable", "istable", "is_calendar_and_gantt", "editable_grid", @@ -344,21 +343,11 @@ "label": "Make Attachments Public by Default" }, { - "default": "0", - "depends_on": "eval: doc.is_submittable", - "fieldname": "queue_in_background", - "fieldtype": "Check", - "label": "Queue in Background" - }, - { - "default": "0", - "depends_on": "eval: doc.is_submittable", - "fetch_from": "doc_type.is_submittable", - "fieldname": "is_submittable", - "fieldtype": "Check", - "label": "Is Submittable", - "read_only": 1 - }, + "default": "0", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" + }, { "fieldname": "default_view", "fieldtype": "Select", @@ -385,7 +374,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-08-30 11:45:16.772277", + "modified": "2022-10-30 23:39:49.628093", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 920df661e3..bdd18cddfa 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -569,9 +569,9 @@ doctype_properties = { "sort_order": "Data", "default_print_format": "Data", "allow_copy": "Check", - "is_submittable": "Check", "istable": "Check", "quick_entry": "Check", + "queue_in_background": "Check", "editable_grid": "Check", "max_attachments": "Int", "make_attachments_public": "Check", From b0f3016c760e6d4c68d2a6b5fd7eba837f235b5d Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 31 Oct 2022 00:52:31 +0530 Subject: [PATCH 061/100] chore: add confirm dialog for unlock doc button --- frappe/core/doctype/submission_queue/submission_queue.js | 4 +++- frappe/core/doctype/submission_queue/submission_queue.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 1f652facd8..414c8c9ee0 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -5,7 +5,9 @@ frappe.ui.form.on("Submission Queue", { refresh: function (frm) { if (frm.doc.status === "Queued") { frm.add_custom_button(__("Unlock Reference Document"), () => { - frm.call("unlock_doc"); + frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => { + frm.call("unlock_doc"); + }); }); } }, diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 37d07f3239..3bb74318f0 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -13,6 +13,7 @@ from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.data import cint class SubmissionQueue(Document): @@ -97,7 +98,7 @@ class SubmissionQueue(Document): frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - if time_diff_in_seconds(now(), get_user_last_request_time(session_id)) < 60: + if cint(time_diff_in_seconds(now(), get_user_last_request_time(session_id))) <= 60: frappe.publish_realtime( "msgprint", { From 51ae7afdee6fd4b07e4fdd373c231feaeee16c08 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 31 Oct 2022 12:29:56 +0530 Subject: [PATCH 062/100] feat: reload listener for form --- .../submission_queue/submission_queue.py | 2 ++ frappe/public/js/frappe/form/form.js | 32 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 3bb74318f0..c0748ebf58 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -120,6 +120,8 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + frappe.publish_realtime(f"reload_doc_{self.ref_doctype}_{self.ref_docname}") + def _unlock_reference_doc(self): try: job = Job.fetch(self.job_id, connection=get_redis_conn()) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 6999edbea2..8027f32454 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -397,6 +397,8 @@ frappe.ui.form.Form = class FrappeForm { // set the doc this.doc = frappe.get_doc(this.doctype, this.docname); + if (!this.doc.__islocal) this.setup_reload_listener(); + // check permissions this.fetch_permissions(); if (!this.has_read_permission()) { @@ -1977,6 +1979,19 @@ frappe.ui.form.Form = class FrappeForm { }); } + setup_reload_listener() { + let doctype = this.doctype; + let docname = this.docname; + let listener_name = `reload_doc_${doctype}_${docname}`; + + frappe.realtime.off(listener_name); + frappe.realtime.on(listener_name, () => { + if (frappe.get_route_str() === `Form/${doctype}/${docname}`) { + this.reload_doc(); + } + }); + } + // Filters fields from the reference doctype and sets them as options for a Select field set_fields_as_options( fieldname, @@ -2039,6 +2054,8 @@ frappe.ui.form.Form = class FrappeForm { } show_submission_queue_banner() { + let wrapper = this.layout.wrapper.find(".submission-queue-banner"); + if ( !( this.meta.is_submittable && @@ -2046,10 +2063,15 @@ frappe.ui.form.Form = class FrappeForm { !this.doc.__islocal && this.doc.docstatus === 0 ) - ) - return; + ) { + if (wrapper.length) { + wrapper.hide(); + wrapper.html(""); + } + + return; + } - let wrapper = this.layout.wrapper.find(".submission-queue-banner"); if (!wrapper.length) { wrapper = $('
'); this.layout.wrapper.prepend(wrapper); @@ -2070,7 +2092,7 @@ frappe.ui.form.Form = class FrappeForm { col_width = 3; failed_link = `
${__( - "Latest Failed Submission" + "Previous Failed Submission" )}
`; } @@ -2081,7 +2103,7 @@ frappe.ui.form.Form = class FrappeForm { ${__("Submission Status:")}
- ${__("Latest Submission")} + ${__("Previous Submission")}
${failed_link}
From e725ae333eaa0f7432b45abe6abdd915c828fcd1 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 1 Nov 2022 15:16:00 +0530 Subject: [PATCH 063/100] fix: dont repeatedly use get_doc for unlocking reference doc * chore: add weird behaviour note on unlock_doc method --- .../core/doctype/submission_queue/submission_queue.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index c0748ebf58..7156de8eee 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -130,8 +130,10 @@ class SubmissionQueue(Document): # assuming the job failed here (?) status = "failed" + queued_doc = self.queued_doc + # Job finished successfully however action was never completed (?) - if status == "finished" and self.queued_doc.docstatus != 1: + if status == "finished" and queued_doc.docstatus == 0: status = "failed" # Checking if job is queue to be executed/executing @@ -140,12 +142,16 @@ class SubmissionQueue(Document): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): - self.queued_doc.unlock() + queued_doc.unlock() frappe.db.set_value("Submission Queue", self.name, "status", "Failed", update_modified=False) frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() def unlock_doc(self): + # NOTE: this can lead to some weird unlocking/locking behaviours. + # for example: hitting unlock on a submission could lead to unlocking of another submission + # of the same reference document. + if self.status != "Queued": return From 074d880dceebdb2fdb83553030961f6b074700e1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 7 Nov 2022 03:19:15 +0530 Subject: [PATCH 064/100] test: Testing for completion --- .../submission_queue/test_submission_queue.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index 3ec99f06b2..51ac15db86 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,21 +1,47 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt +import time +import typing + import frappe -from frappe.tests.utils import FrappeTestCase +from frappe.tests.utils import FrappeTestCase, timeout from frappe.utils.background_jobs import get_queue +if typing.TYPE_CHECKING: + from rq.job import Job + class TestSubmissionQueue(FrappeTestCase): queue = get_queue(qtype="default") + @timeout(seconds=20) + def check_status(self, job: "Job", status, wait=True): + if wait: + while True: + if job.is_queued or job.is_started: + time.sleep(0.2) + else: + break + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) + def test_queue_operation(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.core.doctype.submission_queue.submission_queue import queue_submission - doc = frappe.get_doc({"doctype": "ToDo", "description": "Something"}).insert() - queue_submission(doc, "submit") + doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) + doc.insert() + + d = frappe.new_doc("Test Submission Queue") + d.update({"some_fieldname": "Random"}) + d.insert() + + queue_submission(d, "submit") submission_queue = frappe.get_last_doc("Submission Queue") # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) self.assertIn(job.get_status(refresh=True), ("queued", "started")) + + # Test completion + self.check_status(job, status="finished") From d84c4b28eb23adbde0b93ffd8d29a59e13fb5cd8 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 7 Nov 2022 17:10:25 +0530 Subject: [PATCH 065/100] feat: Adding exception to submisison queue while unlocking doc --- .../doctype/submission_queue/submission_queue.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 7156de8eee..4aa51df746 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -5,6 +5,7 @@ from datetime import datetime from rq.exceptions import NoSuchJobError from rq.job import Job +from rq.registry import FailedJobRegistry import frappe from frappe import _ @@ -12,7 +13,7 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_queue, get_redis_conn from frappe.utils.data import cint @@ -68,7 +69,6 @@ class SubmissionQueue(Document): self, to_be_queued_doc: Document, action_for_queuing: str, enqueued_by_session_id: str ): _action = action_for_queuing.lower() - if _action == "update": _action = "submit" @@ -143,7 +143,15 @@ class SubmissionQueue(Document): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): queued_doc.unlock() - frappe.db.set_value("Submission Queue", self.name, "status", "Failed", update_modified=False) + values = {"status": "Failed"} + + # Defining job exception when unlocking document. + registry = FailedJobRegistry(queue=get_queue(qtype="default")) + for job_id in registry.get_job_ids(): + if job_id == self.job_id: + values = {"status": "Failed", "exception": job.exc_info} + + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() From 4ec69e9384813a2c0b6b4272d55fdc43892078e9 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 8 Nov 2022 12:55:33 +0530 Subject: [PATCH 066/100] refactor: using after commit in enqueuing & removed session time logic --- .../submission_queue/submission_queue.py | 55 +++++++------------ 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4aa51df746..1153a9a8a2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,9 +1,7 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt -from datetime import datetime - -from rq.exceptions import NoSuchJobError +from rq import get_current_job from rq.job import Job from rq.registry import FailedJobRegistry @@ -12,7 +10,7 @@ from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.monitor import add_data_to_monitor -from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds +from frappe.utils import now, time_diff_in_seconds from frappe.utils.background_jobs import get_queue, get_redis_conn from frappe.utils.data import cint @@ -38,10 +36,9 @@ class SubmissionQueue(Document): table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) - def insert(self, to_be_queued_doc: Document, action: str, session_id: str): + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action - self.enqueued_by_session_id = session_id super().insert(ignore_permissions=True) def lock(self): @@ -51,23 +48,24 @@ class SubmissionQueue(Document): self.queued_doc.unlock() def after_insert(self): - job = self.queue_action( + self.queue_action( "background_submission", to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, - enqueued_by_session_id=self.enqueued_by_session_id, timeout=600, + enqueue_after_commit=True, ) + + def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): + current_job = get_current_job() + + # Set the job id for that submission doctype frappe.db.set_value( self.doctype, self.name, - {"job_id": job.id}, + {"job_id": current_job.id}, update_modified=False, ) - - def background_submission( - self, to_be_queued_doc: Document, action_for_queuing: str, enqueued_by_session_id: str - ): _action = action_for_queuing.lower() if _action == "update": _action = "submit" @@ -82,9 +80,9 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - self.notify(values["status"], action_for_queuing, enqueued_by_session_id) + self.notify(values["status"], action_for_queuing) - def notify(self, submission_status: str, action: str, session_id: str): + def notify(self, submission_status: str, action: str): if submission_status == "Failed": doctype = self.doctype docname = self.name @@ -98,7 +96,7 @@ class SubmissionQueue(Document): frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - if cint(time_diff_in_seconds(now(), get_user_last_request_time(session_id))) <= 60: + if cint(time_diff_in_seconds(self.created_at, now())) <= 60: frappe.publish_realtime( "msgprint", { @@ -120,13 +118,9 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - frappe.publish_realtime(f"reload_doc_{self.ref_doctype}_{self.ref_docname}") - def _unlock_reference_doc(self): - try: - job = Job.fetch(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - except NoSuchJobError: + job_id = frappe.db.get_value(self.doctype, self.name, "job_id") + if not job_id: # assuming the job failed here (?) status = "failed" @@ -147,8 +141,10 @@ class SubmissionQueue(Document): # Defining job exception when unlocking document. registry = FailedJobRegistry(queue=get_queue(qtype="default")) - for job_id in registry.get_job_ids(): - if job_id == self.job_id: + # If job id is None then this job won't exist in the failed registry + for jid in registry.get_job_ids(): + if jid == job_id: + job = Job.fetch(job_id, connection=get_redis_conn()) values = {"status": "Failed", "exception": job.exc_info} frappe.db.set_value(self.doctype, self.name, values, update_modified=False) @@ -166,21 +162,12 @@ class SubmissionQueue(Document): self._unlock_reference_doc() -def get_user_last_request_time(session_id): - return ( - frappe.cache() - .hget("session", session_id) - .get("data", {}) - .get("last_updated", datetime.min.strftime(DATETIME_FORMAT)) - ) - - def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.ref_doctype = doc.doctype queue.ref_docname = doc.name - queue.insert(doc, action, frappe.session.sid) + queue.insert(doc, action) frappe.msgprint( _("Queued for Submission. You can track the progress over {0}.").format( From 1c44a65ce8d6193b8b5e61f83f320a303cd9da72 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 8 Nov 2022 20:30:37 +0530 Subject: [PATCH 067/100] test: fixing tests --- .../submission_queue/test_submission_queue.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index 51ac15db86..c057bd22e1 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -29,19 +29,23 @@ class TestSubmissionQueue(FrappeTestCase): from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.core.doctype.submission_queue.submission_queue import queue_submission - doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) - doc.insert() + if not frappe.db.table_exists("Test Submission Queue", cached=False): + doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) + doc.insert() d = frappe.new_doc("Test Submission Queue") d.update({"some_fieldname": "Random"}) d.insert() + frappe.db.commit() queue_submission(d, "submit") + frappe.db.commit() + + # Waiting for execution + time.sleep(4) submission_queue = frappe.get_last_doc("Submission Queue") # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) - self.assertIn(job.get_status(refresh=True), ("queued", "started")) - # Test completion self.check_status(job, status="finished") From bbad2b19ebdd76e31010674116aff37861df5ac5 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 8 Nov 2022 21:32:28 +0530 Subject: [PATCH 068/100] feat: Adding more data to monitor --- frappe/core/doctype/submission_queue/submission_queue.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 1153a9a8a2..ef81cff2c6 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -72,7 +72,12 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - add_data_to_monitor(doctype=to_be_queued_doc.doctype, action=_action) + add_data_to_monitor( + doctype=to_be_queued_doc.doctype, + action=_action, + execution_time=cint(time_diff_in_seconds(now(), self.created_at)), + enqueued_by=self.enqueued_by, + ) values = {"status": "Finished"} except Exception: values = {"status": "Failed", "exception": frappe.get_traceback()} @@ -96,7 +101,7 @@ class SubmissionQueue(Document): frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - if cint(time_diff_in_seconds(self.created_at, now())) <= 60: + if cint(time_diff_in_seconds(now(), self.created_at)) <= 60: frappe.publish_realtime( "msgprint", { From ad18f82007831097992be6df33d31ba430aff162 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 9 Nov 2022 01:10:17 +0530 Subject: [PATCH 069/100] feat: Added execution time to monitor --- .../doctype/submission_queue/submission_queue.json | 2 +- .../core/doctype/submission_queue/submission_queue.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 00e841b926..7d2a1e0703 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -86,7 +86,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-30 19:22:20.998753", + "modified": "2022-11-09 01:01:27.185383", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index ef81cff2c6..063b80f9e9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -74,8 +74,9 @@ class SubmissionQueue(Document): getattr(to_be_queued_doc, _action)() add_data_to_monitor( doctype=to_be_queued_doc.doctype, + docname=to_be_queued_doc.name, action=_action, - execution_time=cint(time_diff_in_seconds(now(), self.created_at)), + execution_time=time_diff_in_seconds(now(), self.created_at), enqueued_by=self.enqueued_by, ) values = {"status": "Finished"} @@ -100,13 +101,14 @@ class SubmissionQueue(Document): message = message.format( frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - - if cint(time_diff_in_seconds(now(), self.created_at)) <= 60: + time_diff = time_diff_in_seconds(now(), self.created_at) + if cint(time_diff) <= 60: frappe.publish_realtime( "msgprint", { "message": message - + f". View it here", + + f". View it here" + + f". Execution time {round(float(time_diff), 2)} seconds", "alert": True, "indicator": "red" if submission_status == "Failed" else "green", }, @@ -128,7 +130,6 @@ class SubmissionQueue(Document): if not job_id: # assuming the job failed here (?) status = "failed" - queued_doc = self.queued_doc # Job finished successfully however action was never completed (?) From 668a730788261ee44ab2c5b8448e38e94910a9ad Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 11 Nov 2022 16:24:39 +0530 Subject: [PATCH 070/100] fix: avoid patching QB if already patched --- .../server_script/test_server_script.py | 22 +++++++++++++++++++ frappe/utils/safe_exec.py | 9 +++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 7002de9691..3abc53bd52 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -211,3 +211,25 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run() """ script.save() script.execute_method() + + def test_scripts_all_the_way_down(self): + # why not + script = frappe.get_doc( + doctype="Server Script", + name="test_nested_scripts_1", + script_type="API", + api_method="test_nested_scripts_1", + script=f"""log("nothing")""", + ) + script.insert() + script.execute_method() + + script = frappe.get_doc( + doctype="Server Script", + name="test_nested_scripts_2", + script_type="API", + api_method="test_nested_scripts_2", + script=f"""frappe.call("test_nested_scripts_1")""", + ) + script.insert() + script.execute_method() diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 04aa134d39..78c205d947 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -269,12 +269,15 @@ def call_with_form_dict(function, kwargs): @contextmanager def patched_qb(): + require_patching = isinstance(frappe.qb.terms, types.ModuleType) try: - _terms = frappe.qb.terms - frappe.qb.terms = _flatten(frappe.qb.terms) + if require_patching: + _terms = frappe.qb.terms + frappe.qb.terms = _flatten(frappe.qb.terms) yield finally: - frappe.qb.terms = _terms + if require_patching: + frappe.qb.terms = _terms @lru_cache From bfab7191543961c6cb77fe267063877c31b616ce Mon Sep 17 00:00:00 2001 From: jll-02 <63648645+jll-02@users.noreply.github.com> Date: Fri, 11 Nov 2022 13:41:45 +0100 Subject: [PATCH 071/100] fix(security): prevent xss attack in search (#18847) --- frappe/templates/includes/navbar/navbar_search.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/templates/includes/navbar/navbar_search.html b/frappe/templates/includes/navbar/navbar_search.html index 67b4d6505a..2602fe1f8c 100644 --- a/frappe/templates/includes/navbar/navbar_search.html +++ b/frappe/templates/includes/navbar/navbar_search.html @@ -2,8 +2,8 @@
    • -{% endif %} \ No newline at end of file +{% endif %} From a7377d23fc1b3b4bf7f846aec5d803bc0e4fabb5 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 11 Nov 2022 19:54:36 +0530 Subject: [PATCH 072/100] refactor!: Drop deprecated functionality (#18815) --- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/core/doctype/user/user.py | 2 +- frappe/database/database.py | 102 ++---------------- .../doctype/newsletter/test_newsletter.py | 2 +- frappe/patches/v13_0/queryreport_columns.py | 2 +- frappe/patches/v14_0/remove_db_aggregation.py | 2 +- frappe/tests/test_db.py | 17 --- 7 files changed, 15 insertions(+), 114 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index acc5c4871d..f4760ec3c6 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -329,7 +329,7 @@ class DocType(Document): "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name) ) for p in parent_list: - frappe.db.update("DocType", p.parent, {}, for_update=False) + frappe.db.set_value("DocType", p.parent, {}, for_update=False) def scrub_field_names(self): """Sluggify fieldnames if not set from Label.""" diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 092f7fa45d..3dc43ccc33 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -471,7 +471,7 @@ class User(Document): frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False) # set email - frappe.db.update("User", new_name, "email", new_name) + frappe.db.set_value("User", new_name, "email", new_name) def append_roles(self, *roles): """Add roles to user""" diff --git a/frappe/database/database.py b/frappe/database/database.py index 47ca451289..dfcc9dfe58 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -30,7 +30,7 @@ from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.utils import cast as cast_fieldtype from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool -from frappe.utils.deprecations import deprecated, deprecation_warning +from frappe.utils.deprecations import deprecated IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") @@ -147,12 +147,11 @@ class Database: self, query: Query, values: QueryValues = EmptyQueryValues, + *, as_dict=0, as_list=0, - formatted=0, debug=0, ignore_ddl=0, - as_utf8=0, auto_commit=0, update=None, explain=False, @@ -165,10 +164,8 @@ class Database: :param values: Tuple / List / Dict of values to be escaped and substituted in the query. :param as_dict: Return as a dictionary. :param as_list: Always return as a list. - :param formatted: Format values like date etc. :param debug: Print query and `EXPLAIN` in debug log. :param ignore_ddl: Catch exception if table, column missing. - :param as_utf8: Encode values as UTF 8. :param auto_commit: Commit after executing the query. :param update: Update this dict to all rows (if returned `as_dict`). :param run: Returns query without executing it if False. @@ -274,20 +271,15 @@ class Database: if pluck: return [r[0] for r in self.last_result] - if as_utf8: - deprecation_warning("as_utf8 parameter is deprecated and will be removed in version 15.") - if formatted: - deprecation_warning("formatted parameter is deprecated and will be removed in version 15.") - # scrub output if required if as_dict: - ret = self.fetch_as_dict(formatted, as_utf8) + ret = self.fetch_as_dict() if update: for r in ret: r.update(update) return ret - elif as_list or as_utf8: - return self.convert_to_lists(self.last_result, formatted, as_utf8) + elif as_list: + return self.convert_to_lists(self.last_result) return self.last_result def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: @@ -394,62 +386,27 @@ class Database: ): raise ImplicitCommitError("This statement can cause implicit commit") - def fetch_as_dict(self, formatted=0, as_utf8=0) -> list[frappe._dict]: + def fetch_as_dict(self) -> list[frappe._dict]: """Internal. Converts results to dict.""" result = self.last_result if result: keys = [column[0] for column in self._cursor.description] - if not as_utf8: - return [frappe._dict(zip(keys, row)) for row in result] - - ret = [] - for r in result: - values = [] - for value in r: - if as_utf8 and isinstance(value, str): - value = value.encode("utf-8") - values.append(value) - - ret.append(frappe._dict(zip(keys, values))) - return ret + return [frappe._dict(zip(keys, row)) for row in result] @staticmethod def clear_db_table_cache(query): if query and is_query_type(query, ("drop", "create")): frappe.cache().delete_key("db_tables") - @staticmethod - def needs_formatting(result, formatted): - """Returns true if the first row in the result has a Date, Datetime, Long Int.""" - if result and result[0]: - for v in result[0]: - if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)): - return True - if formatted and isinstance(v, (int, float)): - return True - - return False - def get_description(self): """Returns result metadata.""" return self._cursor.description @staticmethod - def convert_to_lists(res, formatted=0, as_utf8=0): + def convert_to_lists(res): """Convert tuple output to lists (internal).""" - if not as_utf8: - return [[value for value in row] for row in res] - - nres = [] - for r in res: - nr = [] - for val in r: - if as_utf8 and isinstance(val, str): - val = val.encode("utf-8") - nr.append(val) - nres.append(nr) - return nres + return [[value for value in row] for row in res] def get(self, doctype, filters=None, as_dict=True, cache=False): """Returns `get_value` with fieldname='*'""" @@ -849,11 +806,6 @@ class Database: ).run(debug=debug, run=run, as_dict=as_dict) return {} - @deprecated - def update(self, *args, **kwargs): - """Update multiple values. Alias for `set_value`.""" - return self.set_value(*args, **kwargs) - def set_value( self, dt, @@ -879,7 +831,6 @@ class Database: :param modified_by: Set this user as `modified_by`. :param update_modified: default True. Set as false, if you don't want to update the timestamp. :param debug: Print the query in the developer / js console. - :param for_update: [DEPRECATED] This function now performs updates in single query, locking is not required. """ is_single_doctype = not (dn and dt != dn) to_update = field if isinstance(field, dict) else {field: val} @@ -889,9 +840,6 @@ class Database: modified_by = modified_by or frappe.session.user to_update.update({"modified": modified, "modified_by": modified_by}) - if for_update: - deprecation_warning("for_update parameter is deprecated and will be removed in v15.") - if is_single_doctype: frappe.db.delete( "Singles", filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug @@ -922,32 +870,6 @@ class Database: if dt in self.value_cache: del self.value_cache[dt] - @staticmethod - @deprecated - def set(doc, field, val): - """Set value in document. **Avoid**""" - doc.db_set(field, val) - - @deprecated - def touch(self, doctype, docname): - """Update the modified timestamp of this document.""" - modified = now() - DocType = frappe.qb.DocType(doctype) - frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run() - return modified - - @staticmethod - def set_temp(value): - """Set a temperory value and return a key.""" - key = frappe.generate_hash() - frappe.cache().hset("temp", key, value) - return key - - @staticmethod - def get_temp(key): - """Return the temperory value and delete it.""" - return frappe.cache().hget("temp", key) - def set_global(self, key, val, user="__global"): """Save a global key value. Global values will be automatically set if they match fieldname.""" self.set_default(key, val, user) @@ -1107,7 +1029,7 @@ class Database: return getdate(date).strftime("%Y-%m-%d") @staticmethod - def format_datetime(datetime): + def format_datetime(datetime): # noqa: F811 if not datetime: return FallBackDateTimeStr @@ -1251,10 +1173,6 @@ class Database: """ return self.sql_ddl(f"truncate `{get_table_name(doctype)}`") - @deprecated - def clear_table(self, doctype): - return self.truncate(doctype) - def get_last_created(self, doctype): last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc") if last_record: diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index f80d218bca..a25b6bda02 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -74,7 +74,7 @@ class TestNewsletterMixin: ).insert(ignore_if_duplicate=True) except Exception: frappe.db.rollback(save_point=savepoint) - frappe.db.update(doctype, email_filters, "unsubscribed", 0) + frappe.db.set_value(doctype, email_filters, "unsubscribed", 0) frappe.db.release_savepoint(savepoint) diff --git a/frappe/patches/v13_0/queryreport_columns.py b/frappe/patches/v13_0/queryreport_columns.py index 3081823db6..e9176952d4 100644 --- a/frappe/patches/v13_0/queryreport_columns.py +++ b/frappe/patches/v13_0/queryreport_columns.py @@ -15,4 +15,4 @@ def execute(): if isinstance(data, list): # double escape braces jstr = f'{{"columns":{jstr}}}' - frappe.db.update("Report", record["name"], "json", jstr) + frappe.db.set_value("Report", record["name"], "json", jstr) diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py index 4b0a58c2d6..cff2b583ce 100644 --- a/frappe/patches/v14_0/remove_db_aggregation.py +++ b/frappe/patches/v14_0/remove_db_aggregation.py @@ -32,4 +32,4 @@ def execute(): for agg in ["avg", "max", "min", "sum"]: script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script) - frappe.db.update("Server Script", name, "script", script) + frappe.db.set_value("Server Script", name, "script", script) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index d8259975da..3962cc746d 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -791,23 +791,6 @@ class TestDBSetValue(FrappeTestCase): cached_doc = frappe.get_cached_doc(self.todo2.doctype, self.todo2.name) self.assertEqual(cached_doc.description, description) - def test_update_alias(self): - args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`") - kwargs = { - "for_update": False, - "modified": None, - "modified_by": None, - "update_modified": True, - "debug": False, - } - - self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update)) - - with patch.object(Database, "set_value") as set_value: - frappe.db.update(*args, **kwargs) - set_value.assert_called_once() - set_value.assert_called_with(*args, **kwargs) - @classmethod def tearDownClass(cls): frappe.db.rollback() From 0c3615caa7585f457edb664e6ade5c17f2d83024 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 11 Nov 2022 21:41:51 +0530 Subject: [PATCH 073/100] ci: remove duplicate rule causing backport failure [skip ci] --- .mergify.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 85b590ba76..b74648a8f5 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -71,15 +71,6 @@ pull_request_rules: assignees: - "{{ author }}" - - name: backport to develop - conditions: - - label="backport develop" - actions: - backport: - branches: - - develop - assignees: - - "{{ author }}" - name: backport to version-13-pre-release conditions: From ba27434d816e934f827063edbf5a2094856ad665 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 11 Nov 2022 21:42:51 +0530 Subject: [PATCH 074/100] refactor: Use safer hashing algorithm for verified_command (#18848) --- frappe/utils/verified_command.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/utils/verified_command.py b/frappe/utils/verified_command.py index 7367643cff..b25e646cf5 100644 --- a/frappe/utils/verified_command.py +++ b/frappe/utils/verified_command.py @@ -16,15 +16,15 @@ def get_signed_params(params): if not isinstance(params, str): params = urlencode(params) - signature = hmac.new(params.encode(), digestmod=hashlib.md5) + signature = hmac.new(params.encode(), digestmod=hashlib.sha512) signature.update(get_secret().encode()) return params + "&_signature=" + signature.hexdigest() def get_secret(): - return frappe.local.conf.get("secret") or str( - frappe.db.get_value("User", "Administrator", "creation") - ) + from frappe.utils.password import get_encryption_key + + return frappe.local.conf.get("secret") or get_encryption_key() def verify_request(): @@ -37,10 +37,10 @@ def verify_request(): if signature_string in query_string: params, signature = query_string.split(signature_string) - given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.md5) + given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.sha512) given_signature.update(get_secret().encode()) - valid_signature = signature == given_signature.hexdigest() + valid_signature = hmac.compare_digest(signature, given_signature.hexdigest()) valid_method = frappe.request.method == "GET" valid_request_data = not (frappe.request.form or frappe.request.data) From 7f3ea7a520cef0ca0091755067af757c8bc0dfdb Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sat, 12 Nov 2022 00:36:15 +0530 Subject: [PATCH 075/100] chore(BaseDocument): simplify `_table_fieldnames` property init --- frappe/model/base_document.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index c6b3707b5c..8edf17a5d5 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -100,12 +100,7 @@ class BaseDocument: if d.get("doctype"): self.doctype = d["doctype"] - self._table_fieldnames = ( - d["_table_fieldnames"] # from cache - if "_table_fieldnames" in d - else {df.fieldname for df in self._get_table_fields()} - ) - + self._table_fieldnames = {df.fieldname for df in self._get_table_fields()} self.update(d) self.dont_update_if_missing = [] From 23fe91dede10981f44118f3953f9546078c43837 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 08:26:37 +0530 Subject: [PATCH 076/100] fix: unlock ref doc --- .../submission_queue/submission_queue.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 063b80f9e9..00c6c86582 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -47,6 +47,15 @@ class SubmissionQueue(Document): def unlock(self): self.queued_doc.unlock() + def update_job_id(self, id): + frappe.db.set_value( + self.doctype, + self.name, + {"job_id": id}, + update_modified=False, + ) + frappe.db.commit() + def after_insert(self): self.queue_action( "background_submission", @@ -58,14 +67,8 @@ class SubmissionQueue(Document): def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): current_job = get_current_job() - # Set the job id for that submission doctype - frappe.db.set_value( - self.doctype, - self.name, - {"job_id": current_job.id}, - update_modified=False, - ) + self.update_job_id(current_job.id) _action = action_for_queuing.lower() if _action == "update": _action = "submit" @@ -107,8 +110,7 @@ class SubmissionQueue(Document): "msgprint", { "message": message - + f". View it here" - + f". Execution time {round(float(time_diff), 2)} seconds", + + f". View it here", "alert": True, "indicator": "red" if submission_status == "Failed" else "green", }, @@ -128,8 +130,13 @@ class SubmissionQueue(Document): def _unlock_reference_doc(self): job_id = frappe.db.get_value(self.doctype, self.name, "job_id") if not job_id: - # assuming the job failed here (?) + # Assuming the job failed here (?) + # Or could be in Queue also since we are setting the job id during job.perform() + # No way to tell status = "failed" + else: + status = Job.fetch(job_id, connection=get_redis_conn()).get_status(refresh=True) + queued_doc = self.queued_doc # Job finished successfully however action was never completed (?) From e8b3f8f7106c79ff0e93f300b62ddb956243c523 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 08:57:48 +0530 Subject: [PATCH 077/100] refactor: only showing unlock button if job id is defined --- .../submission_queue/submission_queue.js | 2 +- .../submission_queue/submission_queue.py | 41 +++++-------------- 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 414c8c9ee0..93d6b981dc 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -3,7 +3,7 @@ frappe.ui.form.on("Submission Queue", { refresh: function (frm) { - if (frm.doc.status === "Queued") { + if (frm.doc.status === "Queued" && frm.doc.job_id) { frm.add_custom_button(__("Unlock Reference Document"), () => { frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => { frm.call("unlock_doc"); diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 00c6c86582..f8496d9aa8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -3,7 +3,6 @@ from rq import get_current_job from rq.job import Job -from rq.registry import FailedJobRegistry import frappe from frappe import _ @@ -11,7 +10,7 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import now, time_diff_in_seconds -from frappe.utils.background_jobs import get_queue, get_redis_conn +from frappe.utils.background_jobs import get_redis_conn from frappe.utils.data import cint @@ -128,37 +127,17 @@ class SubmissionQueue(Document): enqueue_create_notification([notify_to], notification_doc) def _unlock_reference_doc(self): - job_id = frappe.db.get_value(self.doctype, self.name, "job_id") - if not job_id: - # Assuming the job failed here (?) - # Or could be in Queue also since we are setting the job id during job.perform() - # No way to tell - status = "failed" - else: - status = Job.fetch(job_id, connection=get_redis_conn()).get_status(refresh=True) + """ + Only execute if self.job_id is defined. + """ + job = Job.fetch(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) - queued_doc = self.queued_doc - - # Job finished successfully however action was never completed (?) - if status == "finished" and queued_doc.docstatus == 0: - status = "failed" - - # Checking if job is queue to be executed/executing if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) - - # Checking any one of the possible termination statuses - elif status in ("failed", "canceled", "stopped"): - queued_doc.unlock() - values = {"status": "Failed"} - - # Defining job exception when unlocking document. - registry = FailedJobRegistry(queue=get_queue(qtype="default")) - # If job id is None then this job won't exist in the failed registry - for jid in registry.get_job_ids(): - if jid == job_id: - job = Job.fetch(job_id, connection=get_redis_conn()) - values = {"status": "Failed", "exception": job.exc_info} + else: + self.queued_doc.unlock() + values = {"status": "Failed", "exception": job.exc_info} frappe.db.set_value(self.doctype, self.name, values, update_modified=False) frappe.msgprint(_("Document Unlocked")) @@ -169,7 +148,7 @@ class SubmissionQueue(Document): # for example: hitting unlock on a submission could lead to unlocking of another submission # of the same reference document. - if self.status != "Queued": + if (self.status != "Queued") or (not self.job_id): return self._unlock_reference_doc() From 36033d41455c4f1e388ce087b12e6d2db9a6145b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 09:32:49 +0530 Subject: [PATCH 078/100] feat: Added finished condition while unlocking --- frappe/core/doctype/submission_queue/submission_queue.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index f8496d9aa8..b915e25d7d 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -135,6 +135,10 @@ class SubmissionQueue(Document): if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) + elif status == "finished": + self.queued_doc.unlock() + frappe.db.set_value(self.doctype, self.name, {"status": "Finished"}, update_modified=False) + frappe.msgprint(_("Document Unlocked")) else: self.queued_doc.unlock() values = {"status": "Failed", "exception": job.exc_info} From 198bc4275fc46838333e195101f2baa2ab5f830d Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sat, 12 Nov 2022 12:48:13 +0530 Subject: [PATCH 079/100] chore: fully commented, consistently formatted JS boilerplates --- frappe/core/doctype/doctype/boilerplate/controller.js | 8 ++++---- .../core/doctype/doctype/boilerplate/controller_list.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/doctype/boilerplate/controller.js b/frappe/core/doctype/doctype/boilerplate/controller.js index 6d9fb2a514..0e3dcd2e26 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller.js +++ b/frappe/core/doctype/doctype/boilerplate/controller.js @@ -1,8 +1,8 @@ // Copyright (c) {year}, {app_publisher} and contributors // For license information, please see license.txt -frappe.ui.form.on('{doctype}', {{ - // refresh: function(frm) {{ +// frappe.ui.form.on("{doctype}", {{ +// refresh(frm) {{ - // }} -}}); +// }}, +// }}); diff --git a/frappe/core/doctype/doctype/boilerplate/controller_list.js b/frappe/core/doctype/doctype/boilerplate/controller_list.js index b1f6d12008..3740cfa85d 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller_list.js +++ b/frappe/core/doctype/doctype/boilerplate/controller_list.js @@ -1,5 +1,5 @@ /* eslint-disable */ -frappe.listview_settings['{doctype}'] = {{ - // add_fields: ["status"], - // filters:[["status","=", "Open"]] -}}; +// frappe.listview_settings["{doctype}"] = {{ +// add_fields: ["status"], +// filters: [["status","=", "Open"]], +// }}; From 6c01d1d417cbf838e75add091a42bcaa6f5ce48e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 12 Nov 2022 12:02:04 +0530 Subject: [PATCH 080/100] refactor: hmac generation Reduce code duplication --- frappe/tests/test_email.py | 14 ++++++++++++++ frappe/utils/verified_command.py | 17 +++++++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index 13e95c38e6..de0fe00012 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -310,6 +310,20 @@ class TestEmail(FrappeTestCase): email_account.enable_incoming = False +class TestVerifiedRequests(FrappeTestCase): + def test_round_trip(self): + from frappe.utils import set_request + from frappe.utils.verified_command import get_signed_params, verify_request + + test_cases = [{"xyz": "abc"}, {"email": "a@b.com", "user": "xyz"}] + + for params in test_cases: + signed_url = get_signed_params(params) + set_request(method="GET", path="?" + signed_url) + self.assertTrue(verify_request()) + frappe.local.request = None + + if __name__ == "__main__": import unittest diff --git a/frappe/utils/verified_command.py b/frappe/utils/verified_command.py index b25e646cf5..3d6346276d 100644 --- a/frappe/utils/verified_command.py +++ b/frappe/utils/verified_command.py @@ -16,9 +16,8 @@ def get_signed_params(params): if not isinstance(params, str): params = urlencode(params) - signature = hmac.new(params.encode(), digestmod=hashlib.sha512) - signature.update(get_secret().encode()) - return params + "&_signature=" + signature.hexdigest() + signature = _sign_message(params) + return params + "&_signature=" + signature def get_secret(): @@ -35,12 +34,10 @@ def verify_request(): signature_string = "&_signature=" if signature_string in query_string: - params, signature = query_string.split(signature_string) + params, given_signature = query_string.split(signature_string) - given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.sha512) - - given_signature.update(get_secret().encode()) - valid_signature = hmac.compare_digest(signature, given_signature.hexdigest()) + computed_signature = _sign_message(params) + valid_signature = hmac.compare_digest(given_signature, computed_signature) valid_method = frappe.request.method == "GET" valid_request_data = not (frappe.request.form or frappe.request.data) @@ -55,6 +52,10 @@ def verify_request(): return False +def _sign_message(message: str) -> str: + return hmac.new(get_secret().encode(), message.encode(), digestmod=hashlib.sha512).hexdigest() + + def get_url(cmd, params, nonce=None, secret=None): if not nonce: nonce = params From 723a27bda777c3e9ddcab5b84a52155ac6f2ddb5 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:08:29 +0530 Subject: [PATCH 081/100] fix: try except for nosuchjoberror in _unlock_reference_doc * fix: condition for not unlocking reference doc * chore: change id param -> job_id for update_job_id --- .../submission_queue/submission_queue.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b915e25d7d..dcb0548e4e 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -2,6 +2,7 @@ # For license information, please see license.txt from rq import get_current_job +from rq.exceptions import NoSuchJobError from rq.job import Job import frappe @@ -46,11 +47,11 @@ class SubmissionQueue(Document): def unlock(self): self.queued_doc.unlock() - def update_job_id(self, id): + def update_job_id(self, job_id): frappe.db.set_value( self.doctype, self.name, - {"job_id": id}, + {"job_id": job_id}, update_modified=False, ) frappe.db.commit() @@ -65,9 +66,9 @@ class SubmissionQueue(Document): ) def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): - current_job = get_current_job() # Set the job id for that submission doctype - self.update_job_id(current_job.id) + self.update_job_id(get_current_job().id) + _action = action_for_queuing.lower() if _action == "update": _action = "submit" @@ -130,21 +131,25 @@ class SubmissionQueue(Document): """ Only execute if self.job_id is defined. """ - job = Job.fetch(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) + try: + job = Job.fetch(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + except NoSuchJobError: + # assuming job failed over here (?) + status = "failed" if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) - elif status == "finished": - self.queued_doc.unlock() - frappe.db.set_value(self.doctype, self.name, {"status": "Finished"}, update_modified=False) - frappe.msgprint(_("Document Unlocked")) - else: - self.queued_doc.unlock() - values = {"status": "Failed", "exception": job.exc_info} + return - frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - frappe.msgprint(_("Document Unlocked")) + self.queued_doc.unlock() + values = ( + {"status": "Finished"} + if status == "finished" + else {"status": "Failed", "exception": job.exc_info} + ) + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() def unlock_doc(self): @@ -152,7 +157,7 @@ class SubmissionQueue(Document): # for example: hitting unlock on a submission could lead to unlocking of another submission # of the same reference document. - if (self.status != "Queued") or (not self.job_id): + if self.status != "Queued" and not self.job_id: return self._unlock_reference_doc() From b21f9a5b264116fbba42acc598ae91ee0f09e91d Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:10:46 +0530 Subject: [PATCH 082/100] fix: submission queue banner if last_failed_submission and last_submissions are equal, show only last_failed_submission --- .../submission_queue/submission_queue.py | 2 +- frappe/public/js/frappe/form/form.js | 32 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index dcb0548e4e..5e400e8707 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -188,5 +188,5 @@ def get_latest_submissions(doctype, docname): filters = {"ref_doctype": doctype, "ref_docname": docname} return { "latest_submission": frappe.db.get_value(dt, filters), - "latest_failed_submission": frappe.db.get_value(dt, filters.update({"status": "Failed"})), + "latest_failed_submission": frappe.db.get_value(dt, filters | {"status": "Failed"}), } diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8027f32454..fa27eaaf9f 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -2073,7 +2073,7 @@ frappe.ui.form.Form = class FrappeForm { } if (!wrapper.length) { - wrapper = $('
    '); + wrapper = $('
    '); this.layout.wrapper.prepend(wrapper); } @@ -2085,16 +2085,21 @@ frappe.ui.form.Form = class FrappeForm { .then((r) => { if (r.message.latest_submission) { // if we are here that means some submission(s) were queued and are in queued/failed state - wrapper.show(); let col_width = 4; let failed_link = ""; + let submission_label = __("Previous Submission"); + if (r.message.latest_failed_submission) { - col_width = 3; - failed_link = `
    - ${__( - "Previous Failed Submission" - )} -
    `; + if (r.message.latest_failed_submission !== r.message.latest_submission) { + col_width = 3; + failed_link = `
    + ${__( + "Previous Falied Submission" + )} +
    `; + } else { + submission_label = __("Previous Falied Submission"); + } } let html = ` @@ -2103,17 +2108,20 @@ frappe.ui.form.Form = class FrappeForm { ${__("Submission Status:")}
    - ${__("Previous Submission")} + ${submission_label}
    ${failed_link}
    - ${__( - "All Submissions" - )} + ${__( + "All Submissions" + )}
    `; + wrapper.show(); wrapper.html(html); } else { wrapper.hide(); From 3dc376cd5ccc27f2930fc39ee712ac9e647c5766 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:16:05 +0530 Subject: [PATCH 083/100] chore: remove reload listener from form --- frappe/public/js/frappe/form/form.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index fa27eaaf9f..4f119f2551 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -397,8 +397,6 @@ frappe.ui.form.Form = class FrappeForm { // set the doc this.doc = frappe.get_doc(this.doctype, this.docname); - if (!this.doc.__islocal) this.setup_reload_listener(); - // check permissions this.fetch_permissions(); if (!this.has_read_permission()) { @@ -1979,19 +1977,6 @@ frappe.ui.form.Form = class FrappeForm { }); } - setup_reload_listener() { - let doctype = this.doctype; - let docname = this.docname; - let listener_name = `reload_doc_${doctype}_${docname}`; - - frappe.realtime.off(listener_name); - frappe.realtime.on(listener_name, () => { - if (frappe.get_route_str() === `Form/${doctype}/${docname}`) { - this.reload_doc(); - } - }); - } - // Filters fields from the reference doctype and sets them as options for a Select field set_fields_as_options( fieldname, From ad6a11e34c4e6b248905512487ccf1cba2081ccb Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:44:20 +0530 Subject: [PATCH 084/100] feat: queue submission for bulk submit --- .../submission_queue/submission_queue.py | 17 +++++++++-------- frappe/desk/doctype/bulk_update/bulk_update.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 5e400e8707..b7ed9cdcc2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -163,20 +163,21 @@ class SubmissionQueue(Document): self._unlock_reference_doc() -def queue_submission(doc: Document, action: str): +def queue_submission(doc: Document, action: str, alert: bool = True): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) - frappe.msgprint( - _("Queued for Submission. You can track the progress over {0}.").format( - f"here" - ), - indicator="green", - alert=True, - ) + if alert: + frappe.msgprint( + _("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) @frappe.whitelist() diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 1e515bbc47..5521d9583f 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -3,8 +3,10 @@ import frappe from frappe import _ +from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.model.document import Document from frappe.utils import cint +from frappe.utils.scheduler import is_scheduler_inactive class BulkUpdate(Document): @@ -44,8 +46,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): try: message = "" if action == "submit" and doc.docstatus.is_draft(): - doc.submit() - message = _("Submitting {0}").format(doctype) + if doc.meta.queue_in_background and not is_scheduler_inactive(): + queue_submission(doc, action) + message = _("Queuing {0} for Submission").format(doctype) + else: + doc.submit() + message = _("Submitting {0}").format(doctype) elif action == "cancel" and doc.docstatus.is_submitted(): doc.cancel() message = _("Cancelling {0}").format(doctype) From f9a10d32d10fc62231010a7178a8a2992628bf9b Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 16:19:44 +0530 Subject: [PATCH 085/100] fix: use quoted doctype and docname in alert --- frappe/core/doctype/submission_queue/submission_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b7ed9cdcc2..6c0ad70bd9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,6 +1,8 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from urllib.parse import quote + from rq import get_current_job from rq.exceptions import NoSuchJobError from rq.job import Job @@ -110,7 +112,7 @@ class SubmissionQueue(Document): "msgprint", { "message": message - + f". View it here", + + f". View it here", "alert": True, "indicator": "red" if submission_status == "Failed" else "green", }, From b09eb2f317de1982fd25565040f09482db8e39ba Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 16:51:58 +0530 Subject: [PATCH 086/100] fix: use index for ref_docname in submission queue doctype --- frappe/core/doctype/submission_queue/submission_queue.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 7d2a1e0703..d1f66ffa13 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -38,7 +38,8 @@ "in_list_view": 1, "label": "Reference Docname", "options": "ref_doctype", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "fieldname": "status", @@ -86,7 +87,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-11-09 01:01:27.185383", + "modified": "2022-11-12 16:48:37.797232", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", From 1145bab9d89891bac6e3f5031d15410f336ecf0c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 19:24:41 +0530 Subject: [PATCH 087/100] refactor: checking docstatus before asserting failure --- .../doctype/submission_queue/submission_queue.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 6c0ad70bd9..872284d46c 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -133,12 +133,18 @@ class SubmissionQueue(Document): """ Only execute if self.job_id is defined. """ + exc = None try: job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) + exc = job.exc_info except NoSuchJobError: - # assuming job failed over here (?) - status = "failed" + # Document is submitted. + if self.queued_doc.docstatus == 1: + status = "finished" + # Document is not submitted and job doesn't exist. + else: + status = "failed" if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) @@ -146,9 +152,7 @@ class SubmissionQueue(Document): self.queued_doc.unlock() values = ( - {"status": "Finished"} - if status == "finished" - else {"status": "Failed", "exception": job.exc_info} + {"status": "Finished"} if status == "finished" else {"status": "Failed", "exception": exc} ) frappe.db.set_value(self.doctype, self.name, values, update_modified=False) frappe.msgprint(_("Document Unlocked")) From 7c8af96046fa7bb2fa436b35d940885a20e0b215 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 19:39:47 +0530 Subject: [PATCH 088/100] refactor: leaving job status as it is on completion --- .../doctype/submission_queue/submission_queue.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 872284d46c..39f36fa356 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -70,6 +70,9 @@ class SubmissionQueue(Document): def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): # Set the job id for that submission doctype self.update_job_id(get_current_job().id) + import time + + time.sleep(10) _action = action_for_queuing.lower() if _action == "update": @@ -133,18 +136,13 @@ class SubmissionQueue(Document): """ Only execute if self.job_id is defined. """ - exc = None try: job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) exc = job.exc_info except NoSuchJobError: - # Document is submitted. - if self.queued_doc.docstatus == 1: - status = "finished" - # Document is not submitted and job doesn't exist. - else: - status = "failed" + exc = None + status = "failed" if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) From 3759e5bbcd32dfe830b36ce8a5b76a7059f363e1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 19:41:21 +0530 Subject: [PATCH 089/100] refactor: lint fix: removed time.sleep --- frappe/core/doctype/submission_queue/submission_queue.py | 4 ---- frappe/desk/form/save.py | 6 +----- frappe/tests/test_document_locks.py | 7 +++---- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 39f36fa356..2bb4200a87 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -70,10 +70,6 @@ class SubmissionQueue(Document): def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): # Set the job id for that submission doctype self.update_job_id(get_current_job().id) - import time - - time.sleep(10) - _action = action_for_queuing.lower() if _action == "update": _action = "submit" diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 8ea5b120e6..f43031c899 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -19,11 +19,7 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if ( - action == "Submit" - and doc.meta.queue_in_background - and not is_scheduler_inactive() - ): + if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): queue_submission(doc, action) return doc.submit() diff --git a/frappe/tests/test_document_locks.py b/frappe/tests/test_document_locks.py index 2ba606685e..5d19f75050 100644 --- a/frappe/tests/test_document_locks.py +++ b/frappe/tests/test_document_locks.py @@ -27,13 +27,12 @@ class TestDocumentLocks(FrappeTestCase): # Checking for persistant locks across all instances. doc = frappe.get_doc("ToDo", todo.name) - self.assertEquals(doc.is_locked, True) + self.assertEqual(doc.is_locked, True) with self.assertRaises(frappe.DocumentLockedError): doc.description = "Random" doc.save() doc.unlock() - self.assertEquals(doc.is_locked, False) - self.assertEquals(todo.is_locked, False) - + self.assertEqual(doc.is_locked, False) + self.assertEqual(todo.is_locked, False) From 838a52328cd3d0f49fafdbad7c955a5e53d5cfe3 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sat, 12 Nov 2022 23:51:02 +0530 Subject: [PATCH 090/100] fix: hardcode doctype in google oauth callback --- frappe/email/oauth.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 89b6df15d8..f5b60a9f3d 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -132,19 +132,18 @@ def oauth_access(email_account: str, service: str): if not service: frappe.throw(frappe._("No Service is selected. Please select one and try again!")) - doctype = "Email Account" - if service == "GMail": - return authorize_google_access(email_account, doctype) + return authorize_google_access(email_account) raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.") -def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None): +def authorize_google_access(email_account: str, code: str = None): """Facilitates google oauth for email. - This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url + This is invoked 2 times - first time when user clicks `Authorize API Access` for getting the authorization url and second time for setting the refresh and access token in db when google redirects back with oauth code.""" + doctype = "Email Account" oauth_obj = GoogleOAuth("mail") if not code: From f019b4fab610b9417b5ceda325aa954cd8375998 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 14 Nov 2022 15:46:54 +0530 Subject: [PATCH 091/100] build(deps): update caniuse (#18866) [skip ci] --- .github/workflows/ui-tests.yml | 1 - package.json | 24 ++++++++++++------------ yarn.lock | 18 ++++-------------- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 8a78d2e750..1a122a3b12 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -121,7 +121,6 @@ jobs: DB: mariadb - name: Verify yarn.lock - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | cd ~/frappe-bench/apps/frappe yarn install --immutable --immutable-cache --check-cache diff --git a/package.json b/package.json index f92deb16a7..5e6754292b 100644 --- a/package.json +++ b/package.json @@ -22,16 +22,24 @@ "homepage": "https://frappeframework.com", "dependencies": { "@editorjs/editorjs": "2.20.0", + "@frappe/esbuild-plugin-postcss2": "^0.1.3", + "@vue/component-compiler": "^4.2.4", "ace-builds": "^1.4.8", "air-datepicker": "github:frappe/air-datepicker", + "autoprefixer": "10", "awesomplete": "^1.1.5", "bootstrap": "4.5.0", + "chalk": "^2.3.2", + "cliui": "^7.0.4", "cookie": "^0.4.0", "cropperjs": "^1.5.12", "cssnano": "^5.0.0", "driver.js": "^0.9.8", "editorjs-undo": "0.1.6", + "esbuild": "^0.14.29", + "esbuild-plugin-vue3": "^0.3.0", "fast-deep-equal": "^2.0.1", + "fast-glob": "^3.2.5", "frappe-charts": "2.0.0-rc22", "frappe-datatable": "^1.16.4", "frappe-gantt": "^0.6.0", @@ -40,16 +48,20 @@ "jquery": "3.6.0", "js-sha256": "^0.9.0", "jsbarcode": "^3.9.0", + "launch-editor": "^2.2.1", "localforage": "^1.9.0", + "md5": "^2.3.0", "moment": "^2.29.4", "moment-timezone": "^0.5.35", "plyr": "^3.7.2", "popper.js": "^1.16.0", + "postcss": "8", "quill": "2.0.0-dev.4", "quill-image-resize": "^3.0.9", "quill-magic-url": "^3.0.0", "qz-tray": "^2.0.8", "redis": "^3.1.1", + "rtlcss": "^3.2.1", "sass": "^1.53.0", "showdown": "^2.1.0", "snyk": "^1.996.0", @@ -62,18 +74,6 @@ "vue-router": "^4.1.5", "vuedraggable": "^4.1.0", "vuex": "4.0.2", - "@frappe/esbuild-plugin-postcss2": "^0.1.3", - "@vue/component-compiler": "^4.2.4", - "autoprefixer": "10", - "chalk": "^2.3.2", - "cliui": "^7.0.4", - "esbuild": "^0.14.29", - "esbuild-plugin-vue3": "^0.3.0", - "fast-glob": "^3.2.5", - "launch-editor": "^2.2.1", - "md5": "^2.3.0", - "postcss": "8", - "rtlcss": "^3.2.1", "yargs": "^17.5.1" }, "snyk": true, diff --git a/yarn.lock b/yarn.lock index 798300ae61..dc0f79efaa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -532,20 +532,10 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001358: - version "1.0.30001359" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001359.tgz#a1c1cbe1c2da9e689638813618b4219acbd4925e" - integrity sha512-Xln/BAsPzEuiVLgJ2/45IaqD9jShtk3Y33anKb4+yLwQzws3+v6odKfpgES/cDEaZMLzSChpIGdbOYtH9MyuHw== - -caniuse-lite@^1.0.30001196: - version "1.0.30001296" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz" - integrity sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q== - -caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001313: - version "1.0.30001316" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001316.tgz#b44a1f419f82d2e119aa0bbdab5ec15471796358" - integrity sha512-JgUdNoZKxPZFzbzJwy4hDSyGuH/gXz2rN51QmoR8cBQsVo58llD3A0vlRKKRt8FGf5u69P9eQyIH8/z9vN/S0Q== +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001196, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001313, caniuse-lite@^1.0.30001358: + version "1.0.30001431" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001431.tgz" + integrity sha512-zBUoFU0ZcxpvSt9IU66dXVT/3ctO1cy4y9cscs1szkPlcWb6pasYM144GqrUygUbT+k7cmUCW61cvskjcv0enQ== chalk@^1.1.3: version "1.1.3" From feb9190cac2006f84535489c4a6e66782be94f55 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Mon, 14 Nov 2022 15:54:41 +0530 Subject: [PATCH 092/100] fix: check if the doctype exists before adding default logtypes in log settings (#18867) --- frappe/core/doctype/log_settings/log_settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index f1b5d23c4f..f0f2cdaae8 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -69,6 +69,9 @@ class LogSettings(Document): added_logtypes = set() for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items(): if logtype not in existing_logtypes and _supports_log_clearing(logtype): + if not frappe.db.exists("DocType", logtype): + continue + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)}) added_logtypes.add(logtype) From 990a96e48bc534a73f499dc47e3dd4897a8fae94 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 14 Nov 2022 15:55:20 +0530 Subject: [PATCH 093/100] feat: show utilization percent on RQ Worker (#18868) [skip ci] --- frappe/core/doctype/rq_worker/rq_worker.json | 12 +++++++++--- frappe/core/doctype/rq_worker/rq_worker.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/rq_worker/rq_worker.json b/frappe/core/doctype/rq_worker/rq_worker.json index ea65abd482..d9a5a23f67 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.json +++ b/frappe/core/doctype/rq_worker/rq_worker.json @@ -20,7 +20,8 @@ "column_break_12", "birth_date", "last_heartbeat", - "total_working_time" + "total_working_time", + "utilization_percent" ], "fields": [ { @@ -59,7 +60,6 @@ { "fieldname": "successful_job_count", "fieldtype": "Int", - "in_list_view": 1, "label": "Successful Job Count" }, { @@ -102,12 +102,18 @@ { "fieldname": "column_break_12", "fieldtype": "Column Break" + }, + { + "fieldname": "utilization_percent", + "fieldtype": "Percent", + "in_list_view": 1, + "label": "Utilization %" } ], "in_create": 1, "is_virtual": 1, "links": [], - "modified": "2022-09-11 05:02:53.981705", + "modified": "2022-11-14 15:35:32.786012", "modified_by": "Administrator", "module": "Core", "name": "RQ Worker", diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py index b2d1f1209d..3de0c8f7fc 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.py +++ b/frappe/core/doctype/rq_worker/rq_worker.py @@ -1,6 +1,9 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +import datetime +from contextlib import suppress + from rq import Worker import frappe @@ -66,4 +69,11 @@ def serialize_worker(worker: Worker) -> frappe._dict: _comment_count=0, modified=convert_utc_to_user_timezone(worker.last_heartbeat), creation=convert_utc_to_user_timezone(worker.birth_date), + utilization_percent=compute_utilization(worker), ) + + +def compute_utilization(worker: Worker) -> float: + with suppress(Exception): + total_time = (datetime.datetime.utcnow() - worker.birth_date).total_seconds() + return worker.total_working_time / total_time * 100 From 5695346668d9c70becf292fdb57add54131eaa76 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 14 Nov 2022 17:33:28 +0530 Subject: [PATCH 094/100] fix: allow mark tag in texteditor (#18871) --- frappe/utils/html_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/utils/html_utils.py b/frappe/utils/html_utils.py index fa84170330..9885a8a8a9 100644 --- a/frappe/utils/html_utils.py +++ b/frappe/utils/html_utils.py @@ -277,6 +277,7 @@ acceptable_elements = [ "li", "m", "map", + "mark", "menu", "meter", "multicol", From 440825a372f1c533727b81782561a0a476214da9 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 14 Nov 2022 18:15:38 +0530 Subject: [PATCH 095/100] refactor: which > find_executable (#18872) Use shutil from the standard library instead of distutils to find executables in PATH --- frappe/build.py | 3 +-- frappe/commands/utils.py | 7 ++++--- frappe/database/db_manager.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/frappe/build.py b/frappe/build.py index e66da4bd79..b74afa5d06 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -4,7 +4,6 @@ import os import re import shutil import subprocess -from distutils.spawn import find_executable from subprocess import getoutput from tempfile import mkdtemp, mktemp from urllib.parse import urlparse @@ -280,7 +279,7 @@ def check_node_executable(): warn = "⚠️ " if node_version.major < 14: click.echo(f"{warn} Please update your node version to 14") - if not find_executable("yarn"): + if not shutil.which("yarn"): click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn") click.echo() diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index e555f63f41..69f6f43ff3 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -2,7 +2,7 @@ import json import os import subprocess import sys -from distutils.spawn import find_executable +from shutil import which import click @@ -12,6 +12,7 @@ from frappe.coverage import CodeCoverage from frappe.exceptions import SiteNotSpecifiedError from frappe.utils import cint, update_progress_bar +find_executable = which # backwards compatibility DATA_IMPORT_DEPRECATION = ( "[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n" "Use `data-import` command instead to import data via 'Data Import'." @@ -525,7 +526,7 @@ def postgres(context): def _mariadb(): from frappe.database.mariadb.database import MariaDBDatabase - mysql = find_executable("mysql") + mysql = which("mysql") command = [ mysql, "--port", @@ -544,7 +545,7 @@ def _mariadb(): def _psql(): - psql = find_executable("psql") + psql = which("psql") subprocess.run([psql, "-d", frappe.conf.db_name]) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 3dddb7f862..5840158fa1 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -51,12 +51,12 @@ class DbManager: @staticmethod def restore_database(target, source, user, password): import os - from distutils.spawn import find_executable + from shutil import which from frappe.utils import make_esc esc = make_esc("$ ") - pv = find_executable("pv") + pv = which("pv") if pv: pipe = f"{pv} {source} |" From 2971abe517b2d98f35a75544db3ff3953da651a0 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 15 Nov 2022 05:37:18 +0000 Subject: [PATCH 096/100] fix: remove middleware to clear `frappe.local` (#18874) --- frappe/app.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/app.py b/frappe/app.py index 0d7fdc1fe1..8f40fcfa82 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -5,7 +5,6 @@ import logging import os from werkzeug.exceptions import HTTPException, NotFound -from werkzeug.local import LocalManager from werkzeug.middleware.profiler import ProfilerMiddleware from werkzeug.middleware.shared_data import SharedDataMiddleware from werkzeug.wrappers import Request, Response @@ -25,13 +24,10 @@ from frappe.utils import get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot from frappe.website.serve import get_response -local_manager = LocalManager(frappe.local) - _site = None _sites_path = os.environ.get("SITES_PATH", ".") -@local_manager.middleware @Request.application def application(request: Request): response = None From 425e4bf1b3dea2fd1daac6e07172850174f8ae08 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 15 Nov 2022 10:49:02 +0000 Subject: [PATCH 097/100] fix(File): validate `attached_to_*` when saving (#18880) --- frappe/core/doctype/file/file.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 1518c72f95..0e28145c9a 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -78,21 +78,38 @@ class File(Document): self.validate_duplicate_entry() def validate(self): + if self.is_folder: + return + # Ensure correct formatting and type self.file_url = unquote(self.file_url) if self.file_url else "" + self.validate_attachment_references() + # when dict is passed to get_doc for creation of new_doc, is_new returns None # this case is handled inside handle_is_private_changed if not self.is_new() and self.has_value_changed("is_private"): self.handle_is_private_changed() - if not self.is_folder: - self.validate_file_path() - self.validate_file_url() - self.validate_file_on_disk() + self.validate_file_path() + self.validate_file_url() + self.validate_file_on_disk() self.file_size = frappe.form_dict.file_size or self.file_size + def validate_attachment_references(self): + if not self.attached_to_doctype: + return + + if self.attached_to_name and not isinstance(self.attached_to_name, str): + frappe.throw(_("Attached To Name must be a string"), TypeError) + + if not self.attached_to_field: + return + + if not frappe.get_meta(self.attached_to_doctype).has_field(self.attached_to_field): + frappe.throw(_("The fieldname you've specified in Attached To Field is invalid")) + def after_rename(self, *args, **kwargs): for successor in self.get_successors(): setup_folder_path(successor, self.name) From 9b90e620bcac12fdd271de32d6f948aaf89a7443 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 15 Nov 2022 17:16:30 +0530 Subject: [PATCH 098/100] chore: disable flaky test This is - flaky - difficult to find source of flake because of crazy tests - adds little value tbh [skip ci] --- frappe/tests/test_zbg_job_sanity_test.py | 27 ------------------------ 1 file changed, 27 deletions(-) delete mode 100644 frappe/tests/test_zbg_job_sanity_test.py diff --git a/frappe/tests/test_zbg_job_sanity_test.py b/frappe/tests/test_zbg_job_sanity_test.py deleted file mode 100644 index 19dc168c04..0000000000 --- a/frappe/tests/test_zbg_job_sanity_test.py +++ /dev/null @@ -1,27 +0,0 @@ -""" smoak tests to check that all registered background jobs execute without error. - -Note: Filename is intentional to run this test roughly at end. Don't change.""" - -import time - -import frappe -from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs -from frappe.tests.utils import FrappeTestCase, timeout - - -class TestScheduledJobSanity(FrappeTestCase): - def setUp(self): - remove_failed_jobs() - - @timeout(90) - def test_bg_jobs_run(self): - """Enqueue all scheduled jobs, wait for finish and verify that none failed.""" - for scheduled_job_type in frappe.get_all("Scheduled Job Type", pluck="name"): - frappe.get_doc("Scheduled Job Type", scheduled_job_type).enqueue(force=True) - - while RQJob.get_list({"filters": [["RQ Job", "status", "in", ("queued", "started")]]}): - time.sleep(0.5) - - # Check no failed, if failed print full details - failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]}) - self.assertEqual(len(failed_jobs), 0, "Jobs failed: " + str(failed_jobs)) From 9fc330ea6c702515f4862e49cab0fae796420ef6 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 15 Nov 2022 18:45:51 +0530 Subject: [PATCH 099/100] Revert "fix: remove middleware to clear `frappe.local` (#18874)" (#18886) This reverts commit 2971abe517b2d98f35a75544db3ff3953da651a0. --- frappe/app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/app.py b/frappe/app.py index 8f40fcfa82..0d7fdc1fe1 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -5,6 +5,7 @@ import logging import os from werkzeug.exceptions import HTTPException, NotFound +from werkzeug.local import LocalManager from werkzeug.middleware.profiler import ProfilerMiddleware from werkzeug.middleware.shared_data import SharedDataMiddleware from werkzeug.wrappers import Request, Response @@ -24,10 +25,13 @@ from frappe.utils import get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot from frappe.website.serve import get_response +local_manager = LocalManager(frappe.local) + _site = None _sites_path = os.environ.get("SITES_PATH", ".") +@local_manager.middleware @Request.application def application(request: Request): response = None From 0d5d2cf95c52f07660e91116e3684b831f2f6275 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 16 Nov 2022 16:15:31 +0530 Subject: [PATCH 100/100] ci: fix flake8 URL (#18895) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0783e94457..e976230244 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,8 +54,8 @@ repos: hooks: - id: isort - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 hooks: - id: flake8 additional_dependencies: ['flake8-bugbear',]