From 077218156b313c3b488fcfc3b2401969baa0c84d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 22 Nov 2023 16:35:18 +0530 Subject: [PATCH 1/3] fix: always publish progress for bulk action --- frappe/desk/doctype/bulk_update/bulk_update.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 526543e825..797428389c 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -85,5 +85,4 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): def show_progress(docnames, message, i, description): n = len(docnames) - if n >= 10: - frappe.publish_progress(float(i) * 100 / n, title=message, description=description) + frappe.publish_progress(float(i) * 100 / n, title=message, description=description) From 866a0295e34cff6565ad345421f4456233d7e7c0 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 22 Nov 2023 17:31:13 +0530 Subject: [PATCH 2/3] fix: bulk submit/cancel/update in background jobs --- .../desk/doctype/bulk_update/bulk_update.py | 17 ++++++- .../doctype/bulk_update/test_bulk_update.py | 48 +++++++++++++++++++ frappe/tests/test_bot.py | 8 ---- 3 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 frappe/desk/doctype/bulk_update/test_bulk_update.py delete mode 100644 frappe/tests/test_bot.py diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 797428389c..cb3d9c7bc6 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -46,8 +46,23 @@ class BulkUpdate(Document): @frappe.whitelist() def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): - docnames = frappe.parse_json(docnames) + if isinstance(docnames, str): + docnames = frappe.parse_json(docnames) + if len(docnames) < 10: + return _bulk_action(doctype, docnames, action, data) + elif len(docnames) <= 100: + frappe.msgprint(_("Bulk operation is enqueued in background."), alert=True) + frappe.enqueue( + _bulk_action, doctype=doctype, docnames=docnames, action=action, data=data, queue="long" + ) + else: + frappe.throw( + _("Bulk operations only support up to 100 documents."), title=_("Too Many Documents") + ) + + +def _bulk_action(doctype, docnames, action, data): if data: data = frappe.parse_json(data) diff --git a/frappe/desk/doctype/bulk_update/test_bulk_update.py b/frappe/desk/doctype/bulk_update/test_bulk_update.py new file mode 100644 index 0000000000..7611141a0a --- /dev/null +++ b/frappe/desk/doctype/bulk_update/test_bulk_update.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023, Frappe Technologies and Contributors +# See LICENSE + +import time + +import frappe +from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.desk.doctype.bulk_update.bulk_update import submit_cancel_or_update_docs +from frappe.tests.utils import FrappeTestCase, timeout + + +class TestBulkUpdate(FrappeTestCase): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.doctype = new_doctype(is_submittable=1, custom=1).insert().name + frappe.db.commit() + for _ in range(50): + frappe.new_doc(cls.doctype, some_fieldname=frappe.mock("name")).insert() + + @timeout() + def wait_for_assertion(self, assertion): + """Wait till an assertion becomes True""" + while True: + if assertion(): + break + time.sleep(0.2) + + def test_bulk_submit_in_background(self): + unsubmitted = frappe.get_all(self.doctype, {"docstatus": 0}, limit=5, pluck="name") + failed = submit_cancel_or_update_docs(self.doctype, unsubmitted, action="submit") + self.assertEqual(failed, []) + + def check_docstatus(docs, status): + frappe.db.rollback() + matching_docs = frappe.get_all( + self.doctype, {"docstatus": status, "name": ("in", docs)}, pluck="name" + ) + return set(matching_docs) == set(docs) + + unsubmitted = frappe.get_all(self.doctype, {"docstatus": 0}, limit=20, pluck="name") + submit_cancel_or_update_docs(self.doctype, unsubmitted, action="submit") + + self.wait_for_assertion(lambda: check_docstatus(unsubmitted, 1)) + + submitted = frappe.get_all(self.doctype, {"docstatus": 1}, limit=20, pluck="name") + submit_cancel_or_update_docs(self.doctype, submitted, action="cancel") + self.wait_for_assertion(lambda: check_docstatus(submitted, 2)) diff --git a/frappe/tests/test_bot.py b/frappe/tests/test_bot.py deleted file mode 100644 index aead54dd63..0000000000 --- a/frappe/tests/test_bot.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: MIT. See LICENSE - -from frappe.tests.utils import FrappeTestCase - - -class TestBot(FrappeTestCase): - pass From 6cf168a56fde7672a87ebf2090f3283310a10fb3 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 24 Nov 2023 16:27:04 +0530 Subject: [PATCH 3/3] fix: bulk workflow action in background --- .../desk/doctype/bulk_update/bulk_update.py | 14 +++++++--- frappe/model/workflow.py | 28 +++++++++++++++---- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index cb3d9c7bc6..27ffb4ffb8 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -49,16 +49,22 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): if isinstance(docnames, str): docnames = frappe.parse_json(docnames) - if len(docnames) < 10: + if len(docnames) < 20: return _bulk_action(doctype, docnames, action, data) - elif len(docnames) <= 100: + elif len(docnames) <= 500: frappe.msgprint(_("Bulk operation is enqueued in background."), alert=True) frappe.enqueue( - _bulk_action, doctype=doctype, docnames=docnames, action=action, data=data, queue="long" + _bulk_action, + doctype=doctype, + docnames=docnames, + action=action, + data=data, + queue="short", + timeout=1000, ) else: frappe.throw( - _("Bulk operations only support up to 100 documents."), title=_("Too Many Documents") + _("Bulk operations only support up to 500 documents."), title=_("Too Many Documents") ) diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index 0d7ce13d95..cc51a55d90 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -1,6 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import json +from collections import defaultdict from typing import TYPE_CHECKING, Union import frappe @@ -233,17 +234,30 @@ def get_workflow_field_value(workflow_name, field): @frappe.whitelist() def bulk_workflow_approval(docnames, doctype, action): - from collections import defaultdict + docnames = json.loads(docnames) + if len(docnames) < 20: + _bulk_workflow_action(docnames, doctype, action) + elif len(docnames) <= 500: + frappe.msgprint(_("Bulk {0} is enqueued in background.").format(action), alert=True) + frappe.enqueue( + _bulk_workflow_action, + docnames=docnames, + doctype=doctype, + action=action, + queue="short", + timeout=1000, + ) + else: + frappe.throw(_("Bulk approval only support up to 500 documents."), title=_("Too Many Documents")) + + +def _bulk_workflow_action(docnames, doctype, action): # dictionaries for logging failed_transactions = defaultdict(list) successful_transactions = defaultdict(list) - # WARN: message log is cleared - print("Clearing frappe.message_log...") frappe.clear_messages() - - docnames = json.loads(docnames) for (idx, docname) in enumerate(docnames, 1): message_dict = {} try: @@ -308,7 +322,9 @@ def print_workflow_log(messages, title, doctype, indicator): html = f"
{doc}
" msg += html - frappe.msgprint(msg, title=_("Workflow Status"), indicator=indicator, is_minimizable=True) + frappe.msgprint( + msg, title=_("Workflow Status"), indicator=indicator, is_minimizable=True, realtime=True + ) @frappe.whitelist()