From 0c09dd4863c03061f503cba812418d4cac3835a4 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 1 Sep 2022 20:03:43 +0530 Subject: [PATCH 001/167] fix: not joining twice implicitly --- frappe/database/query.py | 56 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 5726b760ee..c638eefc52 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -510,7 +510,7 @@ class Engine: return fields - def set_fields(self, table, fields, **kwargs) -> list: + def set_fields(self, fields, **kwargs) -> list: fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name" if isinstance(fields, list) and None in fields and Field not in fields: return None @@ -579,6 +579,8 @@ class Engine: def join_(self, criterion, fields, table, join): """Handles all join operations on criterion objects""" has_join = False + joined_tables = dict() + if not isinstance(fields, Criterion): for field in fields: # Only perform this bit if foreign doctype in fields @@ -587,35 +589,39 @@ class Engine: and str(field).startswith("`tab") and (f"`tab{table}`" not in str(field)) ): - join_table = table_from_string(str(field)) - if self.fieldname: - criterion = criterion.left_join(join_table).on( - getattr(join_table, "name") == getattr(frappe.qb.DocType(table), self.fieldname) - ) - else: - criterion = criterion.left_join(join_table).on( - getattr(join_table, "parent") == getattr(frappe.qb.DocType(table), "name") - ) has_join = True + join_table = table_from_string(str(field)) + # check for already joined tables + if joined_tables.get("left_join") != join_table: + if self.fieldname: + criterion = criterion.left_join(join_table).on( + getattr(join_table, "name") == getattr(frappe.qb.DocType(table), self.fieldname) + ) + joined_tables["left_join"] = join_table + else: + criterion = criterion.left_join(join_table).on( + getattr(join_table, "parent") == getattr(frappe.qb.DocType(table), "name") + ) + joined_tables["left_join"] = join_table - if has_join: - for idx, field in enumerate(fields): - if not is_pypika_function_object(field): - field = field if isinstance(field, str) else field.get_sql() - if not TABLE_PATTERN.search(str(field)): - fields[idx] = getattr(frappe.qb.DocType(table), field) - else: - field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args] - field.args[0] = getattr(frappe.qb.DocType(table), field.args[0].get_sql()) - fields[idx] = field + if has_join: + for idx, field in enumerate(fields): + if not is_pypika_function_object(field): + field = field if isinstance(field, str) else field.get_sql() + if not TABLE_PATTERN.search(str(field)): + fields[idx] = getattr(frappe.qb.DocType(table), field) + else: + field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args] + field.args[0] = getattr(frappe.qb.DocType(table), field.args[0].get_sql()) + fields[idx] = field if len(self.tables) > 1: primary_table = self.tables.pop(table) for table_object in self.tables.values(): - criterion = getattr(criterion, join)(table_object).on( - table_object.parent == primary_table.name - ) - has_join = True + if joined_tables.get("left_join") != table_object: + criterion = getattr(criterion, join)(table_object).on( + table_object.parent == primary_table.name + ) return criterion, fields @@ -631,7 +637,7 @@ class Engine: self.linked_doctype = None self.fieldname = None - fields = self.set_fields(table, kwargs.get("field_objects") or fields, **kwargs) + fields = self.set_fields(fields, **kwargs) criterion = self.build_conditions(table, filters, **kwargs) join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" criterion, fields = self.join_(criterion=criterion, fields=fields, table=table, join=join) From 6262119eecaa5c64c729a09fcd63396446454b30 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 1 Sep 2022 20:04:04 +0530 Subject: [PATCH 002/167] refactor: removed dead code --- frappe/database/database.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 25e82135df..48b756b60e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -780,13 +780,11 @@ class Database: distinct=False, limit=None, ): - field_objects = [] query = frappe.qb.engine.get_query( table=doctype, filters=filters, orderby=order_by, for_update=for_update, - field_objects=field_objects, fields=fields, distinct=distinct, limit=limit, From 3cb98121a84d701dfa9859cb28ef18ce19c256e6 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 2 Sep 2022 00:16:08 +0530 Subject: [PATCH 003/167] refactor: more PseudoColumn checks :( --- frappe/database/query.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index c638eefc52..f4d046c0ba 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -414,7 +414,10 @@ class Engine: if isinstance(operator_mapping, BuiltinFunctionType): has_primitive_operator = True field = operator_mapping( - *map(lambda field: Field(field.strip()), arg.split(_operator)), + *map( + lambda field: Field(field.strip()) if "`" not in field else PseudoColumn(field.strip()), + arg.split(_operator), + ), ) field = ( From ae18f74311f2418a6c3532e2567edd00262a3d1c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 3 Sep 2022 19:07:15 +0530 Subject: [PATCH 004/167] feat: Added nested set support --- frappe/database/query.py | 40 ++++++++++++++++++++++++++++++++++++++-- frappe/database/utils.py | 6 ++++++ frappe/model/db_query.py | 10 ++-------- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index f4d046c0ba..133f31d494 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -9,11 +9,12 @@ from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder import frappe from frappe import _ -from frappe.database.utils import is_pypika_function_object +from frappe.database.utils import is_pypika_function_object, nested_set_hierarchy from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table, functions from frappe.query_builder.functions import Function, SqlFunctions from frappe.query_builder.utils import PseudoColumn +from frappe.utils import cstr if TYPE_CHECKING: from frappe.query_builder import DocType @@ -188,7 +189,7 @@ OPERATOR_MAP: dict[str, Callable] = { "between": func_between, "is": func_is, "timespan": func_timespan, - # TODO: Add support for nested set + "nested_set": nested_set_hierarchy, # TODO: Add support for custom operators (WIP) - via filters_config hooks } @@ -346,7 +347,42 @@ class Engine: if not isinstance(key, str): conditions = conditions.where(self.make_function_for_filters(key, value)) continue + # Nested set support if isinstance(value, (list, tuple)): + if value in OPERATOR_MAP["nested_set"]: + field = frappe.meta.get_field("name") + ref_doctype = field.options if field else table + lft, rgt = "", "" + lft, rgt = frappe.qb.from_(ref_doctype).select(["lft", "rgt"]).where(Field("name") == value[1]).run() + + if value in ("descendants of", "not descendants of"): + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") > lft) + .where(Field("rgt") < rgt) + .orderby(Field("lft"), order=Order.asc) + .run() + ) + else: + # Get ancestor elements of a DocType with a tree structure + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") < lft) + .where(Field("rgt") > rgt) + .orderby(Field("lft"), order=Order.desc) + .run() + ) + if result: + _value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in result] + _operator = ( + self.OPERATOR_MAP["not in"] + if value in ("not ancestors of", "not descendants of") + else self.OPERATOR_MAP["in"] + ) + return conditions.where(_operator(getattr(table, key), _value)) + _operator = self.OPERATOR_MAP[value[0].casefold()] _value = value[1] if value[1] else ("",) conditions = conditions.where(_operator(getattr(table, key), _value)) diff --git a/frappe/database/utils.py b/frappe/database/utils.py index c1f70d388e..40047ffad7 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -18,6 +18,12 @@ QueryValues = tuple | list | dict | NoneType EmptyQueryValues = object() FallBackDateTimeStr = "0001-01-01 00:00:00.000000" +nested_set_hierarchy = ( + "ancestors of", + "descendants of", + "not ancestors of", + "not descendants of", + ) def is_query_type(query: str, query_type: str | tuple[str]) -> bool: return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index a2f597f7bd..a7de3b1a65 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -28,6 +28,7 @@ from frappe.utils import ( get_timespan_date_range, make_filter_tuple, ) +from frappe.database.utils import nested_set_hierarchy LOCATE_PATTERN = re.compile(r"locate\([^,]+,\s*[`\"]?name[`\"]?\s*\)", flags=re.IGNORECASE) LOCATE_CAST_PATTERN = re.compile( @@ -568,21 +569,14 @@ class DatabaseQuery: can_be_null = True # prepare in condition - if f.operator.lower() in ( - "ancestors of", - "descendants of", - "not ancestors of", - "not descendants of", - ): + if f.operator.lower() in nested_set_hierarchy: values = f.value or "" # TODO: handle list and tuple # if not isinstance(values, (list, tuple)): # values = values.split(",") - field = meta.get_field(f.fieldname) ref_doctype = field.options if field else f.doctype - lft, rgt = "", "" if f.value: lft, rgt = frappe.db.get_value(ref_doctype, f.value, ["lft", "rgt"]) From 520c8503a821f080a61c8ae0aa05d4daa5f260cb Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 5 Sep 2022 16:04:38 +0530 Subject: [PATCH 005/167] refactor: using cached property --- frappe/database/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 133f31d494..6664c28a70 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -349,7 +349,7 @@ class Engine: continue # Nested set support if isinstance(value, (list, tuple)): - if value in OPERATOR_MAP["nested_set"]: + if value in self.OPERATOR_MAP["nested_set"]: field = frappe.meta.get_field("name") ref_doctype = field.options if field else table lft, rgt = "", "" From 38f1a9708a56d3d680476084af7b000dcca6e728 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 20 Sep 2022 01:45:20 +0530 Subject: [PATCH 006/167] refactor: simplified logic --- frappe/database/query.py | 11 ++++++----- frappe/database/utils.py | 13 +++++++------ frappe/model/db_query.py | 5 ++--- frappe/tests/test_query.py | 13 +++++++++++++ 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 6664c28a70..ecd1117ba4 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -9,7 +9,7 @@ from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder import frappe from frappe import _ -from frappe.database.utils import is_pypika_function_object, nested_set_hierarchy +from frappe.database.utils import NESTED_SET_HIERARCHY, is_pypika_function_object from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table, functions from frappe.query_builder.functions import Function, SqlFunctions @@ -189,7 +189,7 @@ OPERATOR_MAP: dict[str, Callable] = { "between": func_between, "is": func_is, "timespan": func_timespan, - "nested_set": nested_set_hierarchy, + "nested_set": NESTED_SET_HIERARCHY, # TODO: Add support for custom operators (WIP) - via filters_config hooks } @@ -353,7 +353,9 @@ class Engine: field = frappe.meta.get_field("name") ref_doctype = field.options if field else table lft, rgt = "", "" - lft, rgt = frappe.qb.from_(ref_doctype).select(["lft", "rgt"]).where(Field("name") == value[1]).run() + lft, rgt = ( + frappe.qb.from_(ref_doctype).select(["lft", "rgt"]).where(Field("name") == value[1]).run() + ) if value in ("descendants of", "not descendants of"): result = ( @@ -618,7 +620,7 @@ class Engine: def join_(self, criterion, fields, table, join): """Handles all join operations on criterion objects""" has_join = False - joined_tables = dict() + joined_tables = {} if not isinstance(fields, Criterion): for field in fields: @@ -651,7 +653,6 @@ class Engine: fields[idx] = getattr(frappe.qb.DocType(table), field) else: field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args] - field.args[0] = getattr(frappe.qb.DocType(table), field.args[0].get_sql()) fields[idx] = field if len(self.tables) > 1: diff --git a/frappe/database/utils.py b/frappe/database/utils.py index 40047ffad7..551931d755 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -18,12 +18,13 @@ QueryValues = tuple | list | dict | NoneType EmptyQueryValues = object() FallBackDateTimeStr = "0001-01-01 00:00:00.000000" -nested_set_hierarchy = ( - "ancestors of", - "descendants of", - "not ancestors of", - "not descendants of", - ) +NESTED_SET_HIERARCHY = ( + "ancestors of", + "descendants of", + "not ancestors of", + "not descendants of", +) + def is_query_type(query: str, query_type: str | tuple[str]) -> bool: return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index a7de3b1a65..de2a73bfb6 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -13,7 +13,7 @@ import frappe.permissions import frappe.share from frappe import _ from frappe.core.doctype.server_script.server_script_utils import get_server_script_map -from frappe.database.utils import FallBackDateTimeStr +from frappe.database.utils import NESTED_SET_HIERARCHY, FallBackDateTimeStr from frappe.model import optional_fields from frappe.model.meta import get_table_columns from frappe.model.utils.user_settings import get_user_settings, update_user_settings @@ -28,7 +28,6 @@ from frappe.utils import ( get_timespan_date_range, make_filter_tuple, ) -from frappe.database.utils import nested_set_hierarchy LOCATE_PATTERN = re.compile(r"locate\([^,]+,\s*[`\"]?name[`\"]?\s*\)", flags=re.IGNORECASE) LOCATE_CAST_PATTERN = re.compile( @@ -569,7 +568,7 @@ class DatabaseQuery: can_be_null = True # prepare in condition - if f.operator.lower() in nested_set_hierarchy: + if f.operator.lower() in NESTED_SET_HIERARCHY: values = f.value or "" # TODO: handle list and tuple diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index ae9513a302..9dc335134a 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -201,3 +201,16 @@ class TestQuery(FrappeTestCase): fields=["name", "`tabNote Seen By`.`user` as seen_by"], ), ) + + self.assertEqual( + frappe.qb.engine.get_query( + "Note", + filters={"name": "Test Note Title"}, + fields=["name", "`tabNote Seen By`.`docstatus`", "`tabNote Seen By`.`user`"], + ).run(as_dict=1), + frappe.get_list( + "Note", + filters={"name": "Test Note Title"}, + fields=["name", "`tabNote Seen By`.`docstatus`", "`tabNote Seen By`.`user`"], + ), + ) From ee3651b103a808c4159ddf959bfe6e3c563125db Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 20 Sep 2022 01:54:41 +0530 Subject: [PATCH 007/167] refactor: separated nested set logic --- frappe/database/query.py | 57 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index ecd1117ba4..cadd16297c 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -284,6 +284,36 @@ class Engine: return conditions + @staticmethod + def get_condition_from_nested_sets(value: list | tuple, table: str): + field = frappe.meta.get_field("name") + ref_doctype = field.options if field else table + lft, rgt = "", "" + lft, rgt = ( + frappe.qb.from_(ref_doctype).select(["lft", "rgt"]).where(Field("name") == value[1]).run() + ) + + if value in ("descendants of", "not descendants of"): + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") > lft) + .where(Field("rgt") < rgt) + .orderby(Field("lft"), order=Order.asc) + .run() + ) + else: + # Get ancestor elements of a DocType with a tree structure + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") < lft) + .where(Field("rgt") > rgt) + .orderby(Field("lft"), order=Order.desc) + .run() + ) + return result + def misc_query(self, table: str, filters: list | tuple = None, **kwargs): """Build conditions using the given Lists or Tuple filters @@ -350,32 +380,7 @@ class Engine: # Nested set support if isinstance(value, (list, tuple)): if value in self.OPERATOR_MAP["nested_set"]: - field = frappe.meta.get_field("name") - ref_doctype = field.options if field else table - lft, rgt = "", "" - lft, rgt = ( - frappe.qb.from_(ref_doctype).select(["lft", "rgt"]).where(Field("name") == value[1]).run() - ) - - if value in ("descendants of", "not descendants of"): - result = ( - frappe.qb.from_(ref_doctype) - .select(Field("name")) - .where(Field("lft") > lft) - .where(Field("rgt") < rgt) - .orderby(Field("lft"), order=Order.asc) - .run() - ) - else: - # Get ancestor elements of a DocType with a tree structure - result = ( - frappe.qb.from_(ref_doctype) - .select(Field("name")) - .where(Field("lft") < lft) - .where(Field("rgt") > rgt) - .orderby(Field("lft"), order=Order.desc) - .run() - ) + result = self.get_condition_from_nested_sets(value, table) if result: _value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in result] _operator = ( From f62d1540cf4a97b9da1c54fb94b75f32a51f5015 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 24 Sep 2022 14:00:55 +0530 Subject: [PATCH 008/167] feat: Background submissions for submittable doctypes --- frappe/core/doctype/doctype/doctype.json | 11 +++++++++-- frappe/desk/form/save.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index bfa91cea75..4e6c4bc80f 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -85,7 +85,8 @@ "website_search_field", "advanced", "engine", - "migration_hash" + "migration_hash", + "bg_submit" ], "fields": [ { @@ -605,6 +606,12 @@ "fieldname": "make_attachments_public", "fieldtype": "Check", "label": "Make Attachments Public by Default" + }, + { + "default": "0", + "fieldname": "bg_submit", + "fieldtype": "Check", + "label": "BgSubmit" } ], "icon": "fa fa-bolt", @@ -687,7 +694,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-08-24 06:42:27.779699", + "modified": "2022-09-24 11:10:26.020900", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index f3e7b6294f..78396f666b 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -17,9 +17,15 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - doc.submit() + if doc.bg_submit and doc.is_submittable: + doc.queue_action("submit", timeout=4000) + else: + doc.submit() else: - doc.save() + if doc.bg_submit and doc.is_submittable: + doc.queue_action("save", timeout=4000) + else: + doc.save() # update recent documents run_onload(doc) From 8b1a01e4b7e914c5b20a7d2fe3491851c0732e7a Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 30 Sep 2022 19:25:06 +0530 Subject: [PATCH 009/167] refactor: changed field name & removed queueing from save --- frappe/core/doctype/doctype/doctype.json | 8 ++++---- frappe/desk/form/save.py | 8 ++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 4e6c4bc80f..15671bc2b8 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -86,7 +86,7 @@ "advanced", "engine", "migration_hash", - "bg_submit" + "submit_in_background" ], "fields": [ { @@ -609,9 +609,9 @@ }, { "default": "0", - "fieldname": "bg_submit", + "fieldname": "submit_in_background", "fieldtype": "Check", - "label": "BgSubmit" + "label": "Submit in background" } ], "icon": "fa fa-bolt", @@ -694,7 +694,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-09-24 11:10:26.020900", + "modified": "2022-09-30 19:18:02.789866", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index d8204337de..71ec91e6ce 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -16,17 +16,13 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] - if doc.docstatus == 1: - if doc.bg_submit and doc.is_submittable: + if doc.meta.submit_in_background and doc.meta.is_submittable: doc.queue_action("submit", timeout=4000) else: doc.submit() else: - if doc.bg_submit and doc.is_submittable: - doc.queue_action("save", timeout=4000) - else: - doc.save() + doc.save() # update recent documents run_onload(doc) From b787a49428d8c0a1aff2ad76e519a6d736b0cf2e Mon Sep 17 00:00:00 2001 From: Aradhya Tripathi <67282231+Aradhya-Tripathi@users.noreply.github.com> Date: Mon, 3 Oct 2022 13:46:06 +0530 Subject: [PATCH 010/167] refactor: removed excess checks Co-authored-by: Ritwik Puri --- frappe/desk/form/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 71ec91e6ce..3cdd4e6f01 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -17,7 +17,7 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if doc.meta.submit_in_background and doc.meta.is_submittable: + if doc.meta.submit_in_background: doc.queue_action("submit", timeout=4000) else: doc.submit() From eae73ae5d45414f7e8b4dce0326abae8b41c9e26 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 3 Oct 2022 17:36:33 +0530 Subject: [PATCH 011/167] feat: log reports for queued submits --- frappe/core/doctype/doctype/doctype.json | 6 +- frappe/core/doctype/queued_submit/__init__.py | 0 .../doctype/queued_submit/queued_submit.js | 8 ++ .../doctype/queued_submit/queued_submit.json | 80 +++++++++++++++++++ .../doctype/queued_submit/queued_submit.py | 8 ++ .../queued_submit/test_queued_submit.py | 9 +++ frappe/desk/form/save.py | 4 +- frappe/model/document.py | 23 ++++++ frappe/utils/background_jobs.py | 1 + 9 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 frappe/core/doctype/queued_submit/__init__.py create mode 100644 frappe/core/doctype/queued_submit/queued_submit.js create mode 100644 frappe/core/doctype/queued_submit/queued_submit.json create mode 100644 frappe/core/doctype/queued_submit/queued_submit.py create mode 100644 frappe/core/doctype/queued_submit/test_queued_submit.py diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 15671bc2b8..b43a9ec3ae 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -58,6 +58,7 @@ "icon", "color", "show_preview_popup", + "submit_in_background", "show_name_in_global_search", "email_settings_sb", "default_email_template", @@ -85,8 +86,7 @@ "website_search_field", "advanced", "engine", - "migration_hash", - "submit_in_background" + "migration_hash" ], "fields": [ { @@ -694,7 +694,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-09-30 19:18:02.789866", + "modified": "2022-10-03 16:14:46.432770", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/core/doctype/queued_submit/__init__.py b/frappe/core/doctype/queued_submit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/queued_submit/queued_submit.js b/frappe/core/doctype/queued_submit/queued_submit.js new file mode 100644 index 0000000000..fec6615884 --- /dev/null +++ b/frappe/core/doctype/queued_submit/queued_submit.js @@ -0,0 +1,8 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Queued Submit', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/queued_submit/queued_submit.json b/frappe/core/doctype/queued_submit/queued_submit.json new file mode 100644 index 0000000000..9cd2dde1b5 --- /dev/null +++ b/frappe/core/doctype/queued_submit/queued_submit.json @@ -0,0 +1,80 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2022-10-03 17:19:13.116028", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "title", + "job_id", + "state", + "start_time", + "created_by", + "error" + ], + "fields": [ + { + "fieldname": "title", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Title", + "reqd": 1 + }, + { + "fieldname": "job_id", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Job Id", + "reqd": 1 + }, + { + "fieldname": "state", + "fieldtype": "Data", + "label": "State" + }, + { + "fieldname": "start_time", + "fieldtype": "Datetime", + "in_list_view": 1, + "label": "Start Time", + "reqd": 1 + }, + { + "fieldname": "created_by", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Created By", + "reqd": 1 + }, + { + "fieldname": "error", + "fieldtype": "Text", + "label": "Error" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2022-10-03 17:28:12.081732", + "modified_by": "Administrator", + "module": "Core", + "name": "Queued Submit", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/queued_submit/queued_submit.py b/frappe/core/doctype/queued_submit/queued_submit.py new file mode 100644 index 0000000000..65ee12a0c3 --- /dev/null +++ b/frappe/core/doctype/queued_submit/queued_submit.py @@ -0,0 +1,8 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class QueuedSubmit(Document): + pass diff --git a/frappe/core/doctype/queued_submit/test_queued_submit.py b/frappe/core/doctype/queued_submit/test_queued_submit.py new file mode 100644 index 0000000000..86c4c803ca --- /dev/null +++ b/frappe/core/doctype/queued_submit/test_queued_submit.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestQueuedSubmit(FrappeTestCase): + pass diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 3cdd4e6f01..05484664a5 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -18,9 +18,10 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - doc.queue_action("submit", timeout=4000) + doc.submit_in_background() else: doc.submit() + else: doc.save() @@ -29,7 +30,6 @@ def savedocs(doc, action): send_updated_docs(doc) add_data_to_monitor(doctype=doc.doctype, action=action) - frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) diff --git a/frappe/model/document.py b/frappe/model/document.py index aa55eac30a..489cb56a19 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,5 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from datetime import datetime import hashlib import json import time @@ -984,6 +985,28 @@ class Document(BaseDocument): elif alert.event == "Method" and method == alert.method: _evaluate_alert(alert) + def submit_in_background(self): + job = self.queue_action("_submit_in_background") + doc = frappe.new_doc("Queued Submit") + doc.title = self.name + doc.state = "Queued" + doc.start_time = datetime.now() + doc.job_id = job.id + doc.created_by = frappe.session.user + doc.insert() + + def _submit_in_background(self): + try: + self.submit() + doc = frappe.get_doc("Queued Submit", self.title) + doc.state = "Submitted" + doc.insert() + except Exception as e: + doc = frappe.get_doc("Queued Submit", self.title) + doc.state = "Failed" + doc.error = str(e) + doc.insert() + @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index e88cd75efb..0834a403da 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -9,6 +9,7 @@ from uuid import uuid4 import redis from redis.exceptions import BusyLoadingError, ConnectionError from rq import Connection, Queue, Worker +from rq.registry import FailedJobRegistry from rq.command import send_stop_job_command from rq.logutils import setup_loghandlers from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed From 9cc826541fe2b27892adfe6c5870c8890b1a76a2 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 02:41:14 +0530 Subject: [PATCH 012/167] fix: fixed functionality --- frappe/core/doctype/queued_submit/__init__.py | 0 .../doctype/queued_submit/queued_submit.js | 8 -- .../doctype/queued_submit/queued_submit.json | 80 ------------------- .../doctype/queued_submit/queued_submit.py | 8 -- .../queued_submit/test_queued_submit.py | 9 --- frappe/model/document.py | 19 ++--- frappe/utils/background_jobs.py | 1 - 7 files changed, 8 insertions(+), 117 deletions(-) delete mode 100644 frappe/core/doctype/queued_submit/__init__.py delete mode 100644 frappe/core/doctype/queued_submit/queued_submit.js delete mode 100644 frappe/core/doctype/queued_submit/queued_submit.json delete mode 100644 frappe/core/doctype/queued_submit/queued_submit.py delete mode 100644 frappe/core/doctype/queued_submit/test_queued_submit.py diff --git a/frappe/core/doctype/queued_submit/__init__.py b/frappe/core/doctype/queued_submit/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/core/doctype/queued_submit/queued_submit.js b/frappe/core/doctype/queued_submit/queued_submit.js deleted file mode 100644 index fec6615884..0000000000 --- a/frappe/core/doctype/queued_submit/queued_submit.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2022, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Queued Submit', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/queued_submit/queued_submit.json b/frappe/core/doctype/queued_submit/queued_submit.json deleted file mode 100644 index 9cd2dde1b5..0000000000 --- a/frappe/core/doctype/queued_submit/queued_submit.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "actions": [], - "allow_rename": 1, - "creation": "2022-10-03 17:19:13.116028", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "title", - "job_id", - "state", - "start_time", - "created_by", - "error" - ], - "fields": [ - { - "fieldname": "title", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Title", - "reqd": 1 - }, - { - "fieldname": "job_id", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Job Id", - "reqd": 1 - }, - { - "fieldname": "state", - "fieldtype": "Data", - "label": "State" - }, - { - "fieldname": "start_time", - "fieldtype": "Datetime", - "in_list_view": 1, - "label": "Start Time", - "reqd": 1 - }, - { - "fieldname": "created_by", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Created By", - "reqd": 1 - }, - { - "fieldname": "error", - "fieldtype": "Text", - "label": "Error" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2022-10-03 17:28:12.081732", - "modified_by": "Administrator", - "module": "Core", - "name": "Queued Submit", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "states": [] -} \ No newline at end of file diff --git a/frappe/core/doctype/queued_submit/queued_submit.py b/frappe/core/doctype/queued_submit/queued_submit.py deleted file mode 100644 index 65ee12a0c3..0000000000 --- a/frappe/core/doctype/queued_submit/queued_submit.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2022, Frappe Technologies and contributors -# For license information, please see license.txt - -# import frappe -from frappe.model.document import Document - -class QueuedSubmit(Document): - pass diff --git a/frappe/core/doctype/queued_submit/test_queued_submit.py b/frappe/core/doctype/queued_submit/test_queued_submit.py deleted file mode 100644 index 86c4c803ca..0000000000 --- a/frappe/core/doctype/queued_submit/test_queued_submit.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2022, Frappe Technologies and Contributors -# See license.txt - -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestQueuedSubmit(FrappeTestCase): - pass diff --git a/frappe/model/document.py b/frappe/model/document.py index 489cb56a19..4e2cac89c1 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from datetime import datetime import hashlib import json import time +from datetime import datetime from werkzeug.exceptions import NotFound @@ -987,25 +987,22 @@ class Document(BaseDocument): def submit_in_background(self): job = self.queue_action("_submit_in_background") - doc = frappe.new_doc("Queued Submit") - doc.title = self.name + doc = frappe.new_doc("Submission Queue") doc.state = "Queued" doc.start_time = datetime.now() - doc.job_id = job.id doc.created_by = frappe.session.user + doc.name = self.doctype + str(self.name) + doc.job_id = job.id doc.insert() def _submit_in_background(self): try: self.submit() - doc = frappe.get_doc("Queued Submit", self.title) - doc.state = "Submitted" - doc.insert() + frappe.db.set_value("Submission Queue", self.doctype + str(self.name), {"state": "Submitted"}) except Exception as e: - doc = frappe.get_doc("Queued Submit", self.title) - doc.state = "Failed" - doc.error = str(e) - doc.insert() + frappe.db.set_value( + "Submission Queue", self.doctype + str(self.name), {"state": "Failed", "error": e} + ) @whitelist.__func__ def _submit(self): diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 0834a403da..e88cd75efb 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -9,7 +9,6 @@ from uuid import uuid4 import redis from redis.exceptions import BusyLoadingError, ConnectionError from rq import Connection, Queue, Worker -from rq.registry import FailedJobRegistry from rq.command import send_stop_job_command from rq.logutils import setup_loghandlers from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed From 198bc39085979dda260c292621a29eb5bcbc787c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 17:17:45 +0530 Subject: [PATCH 013/167] refactor: moved submission from document class --- frappe/core/doctype/report/report.js | 1 - .../core/doctype/submission_queue/__init__.py | 0 .../submission_queue/submission_queue.js | 7 ++ .../submission_queue/submission_queue.json | 78 +++++++++++++++++++ .../submission_queue/submission_queue.py | 44 +++++++++++ .../submission_queue/test_submission_queue.py | 9 +++ frappe/desk/form/save.py | 8 +- frappe/model/document.py | 19 ----- 8 files changed, 143 insertions(+), 23 deletions(-) create mode 100644 frappe/core/doctype/submission_queue/__init__.py create mode 100644 frappe/core/doctype/submission_queue/submission_queue.js create mode 100644 frappe/core/doctype/submission_queue/submission_queue.json create mode 100644 frappe/core/doctype/submission_queue/submission_queue.py create mode 100644 frappe/core/doctype/submission_queue/test_submission_queue.py diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index 0e0bfeea9a..9850dbf98f 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -31,7 +31,6 @@ frappe.ui.form.on("Report", { ); } - if (doc.is_standard === "Yes" && frm.perm[0].write) { frm.add_custom_button( doc.disabled ? __("Enable Report") : __("Disable Report"), diff --git a/frappe/core/doctype/submission_queue/__init__.py b/frappe/core/doctype/submission_queue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js new file mode 100644 index 0000000000..df0833c004 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -0,0 +1,7 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Submission Queue", { + // refresh: function(frm) { + // } +}); diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json new file mode 100644 index 0000000000..638d28d3fe --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -0,0 +1,78 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2022-10-04 00:41:00.028163", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "state", + "start_time", + "created_by", + "job_id", + "ref_doctype", + "ref_docname" + ], + "fields": [ + { + "fieldname": "state", + "fieldtype": "Select", + "label": "State", + "options": "Submitted\nQueued\nFailed", + "read_only": 1 + }, + { + "fieldname": "start_time", + "fieldtype": "Datetime", + "label": "Start time", + "read_only": 1 + }, + { + "fieldname": "created_by", + "fieldtype": "Data", + "label": "Created By", + "read_only": 1 + }, + { + "fieldname": "job_id", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Job Id", + "read_only": 1 + }, + { + "fieldname": "ref_doctype", + "fieldtype": "Link", + "label": "Reference DocType", + "options": "DocType" + }, + { + "fieldname": "ref_docname", + "fieldtype": "Dynamic Link", + "label": "Reference Docname", + "options": "ref_doctype" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2022-10-04 16:16:42.962361", + "modified_by": "Administrator", + "module": "Core", + "name": "Submission Queue", + "naming_rule": "Random", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py new file mode 100644 index 0000000000..4f4a9a8ca4 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -0,0 +1,44 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +from datetime import datetime + +import frappe +from frappe import _ + +# import frappe +from frappe.model.document import Document +from frappe.utils.background_jobs import enqueue + + +class SubmissionQueue(Document): + ... + + +def submit_in_background(doc: Document): + try: + doc.lock() + except frappe.DocumentLockedError: + frappe.throw( + _("This document is currently queued for execution. Please try again"), + title=_("Document Queued"), + exc=frappe.DocumentLockedError, + ) + new_queue = frappe.new_doc("Submission Queue") + new_queue.state = "Queued" + new_queue.start_time = datetime.now() + new_queue.created_by = frappe.session.user + new_queue.ref_doctype = doc.doctype + new_queue.ref_docname = doc.name + new_queue.insert() + job = enqueue(_submit_in_background, name=new_queue.name, doc=doc) + new_queue.job_id = job.id + new_queue.save() + + +def _submit_in_background(name: str, doc: Document): + try: + doc.submit() + frappe.db.set_value("Submission Queue", name, {"state": "Submitted"}) + except Exception as e: + frappe.db.set_value("Submission Queue", name, {"state": "Failed", "error": e}) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py new file mode 100644 index 0000000000..d7547983a2 --- /dev/null +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestSubmissionQueue(FrappeTestCase): + pass diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 05484664a5..5944656a13 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -4,6 +4,7 @@ import json import frappe +from frappe.core.doctype.submission_queue.submission_queue import submit_in_background from frappe.desk.form.load import run_onload from frappe.monitor import add_data_to_monitor @@ -18,10 +19,10 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - doc.submit_in_background() + submit_in_background(doc) + frappe.msgprint(frappe._("Queued"), indicator="green", alert=True) else: doc.submit() - else: doc.save() @@ -30,7 +31,8 @@ def savedocs(doc, action): send_updated_docs(doc) add_data_to_monitor(doctype=doc.doctype, action=action) - frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) + if not doc.meta.submit_in_background: + frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) @frappe.whitelist() diff --git a/frappe/model/document.py b/frappe/model/document.py index 4e2cac89c1..ca27a830d3 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -985,25 +985,6 @@ class Document(BaseDocument): elif alert.event == "Method" and method == alert.method: _evaluate_alert(alert) - def submit_in_background(self): - job = self.queue_action("_submit_in_background") - doc = frappe.new_doc("Submission Queue") - doc.state = "Queued" - doc.start_time = datetime.now() - doc.created_by = frappe.session.user - doc.name = self.doctype + str(self.name) - doc.job_id = job.id - doc.insert() - - def _submit_in_background(self): - try: - self.submit() - frappe.db.set_value("Submission Queue", self.doctype + str(self.name), {"state": "Submitted"}) - except Exception as e: - frappe.db.set_value( - "Submission Queue", self.doctype + str(self.name), {"state": "Failed", "error": e} - ) - @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" From 33acaf7efdb7df363bb59e2391550d4632f58bf1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 17:24:27 +0530 Subject: [PATCH 014/167] fix: unlock document when action is executed --- frappe/core/doctype/submission_queue/submission_queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4f4a9a8ca4..38f0262fce 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -37,6 +37,7 @@ def submit_in_background(doc: Document): def _submit_in_background(name: str, doc: Document): + doc.unlock() try: doc.submit() frappe.db.set_value("Submission Queue", name, {"state": "Submitted"}) From 4bdffe73ddd3f0e74ceac7dbece6cf8e86983230 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 4 Oct 2022 23:06:46 +0530 Subject: [PATCH 015/167] refactor: moved background submit to Submission Queue DocType Co-authored-by: phot0n ritwikpuri5678@gmail.com --- .../submission_queue/submission_queue.json | 60 +++++++++++--- .../submission_queue/submission_queue.py | 82 ++++++++++++------- frappe/desk/form/save.py | 10 +-- 3 files changed, 104 insertions(+), 48 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 638d28d3fe..f2972c5927 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -6,21 +6,17 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ - "state", + "status", "start_time", "created_by", "job_id", + "column_break_5", "ref_doctype", - "ref_docname" + "ref_docname", + "section_break_8", + "message" ], "fields": [ - { - "fieldname": "state", - "fieldtype": "Select", - "label": "State", - "options": "Submitted\nQueued\nFailed", - "read_only": 1 - }, { "fieldname": "start_time", "fieldtype": "Datetime", @@ -44,18 +40,43 @@ "fieldname": "ref_doctype", "fieldtype": "Link", "label": "Reference DocType", - "options": "DocType" + "options": "DocType", + "read_only": 1 }, { "fieldname": "ref_docname", "fieldtype": "Dynamic Link", "label": "Reference Docname", - "options": "ref_doctype" + "options": "ref_doctype", + "read_only": 1 + }, + { + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "in_list_view": 1, + "label": "Status", + "options": "Queued\nCompleted\nFailed", + "read_only": 1 + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "message", + "fieldtype": "Text", + "label": "Message", + "read_only": 1 + }, + { + "fieldname": "section_break_8", + "fieldtype": "Section Break" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-04 16:16:42.962361", + "modified": "2022-10-04 18:41:08.495515", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -74,5 +95,18 @@ ], "sort_field": "modified", "sort_order": "DESC", - "states": [] + "states": [ + { + "color": "Blue", + "title": "Queued" + }, + { + "color": "Red", + "title": "Failed" + }, + { + "color": "Green", + "title": "Completed" + } + ] } \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 38f0262fce..621de6357b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -5,41 +5,63 @@ from datetime import datetime import frappe from frappe import _ - -# import frappe +from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document -from frappe.utils.background_jobs import enqueue class SubmissionQueue(Document): - ... + def insert(self, to_be_queued_doc: Document, action: str): + self.to_be_queued_doc = to_be_queued_doc + self.action_for_queuing = action + super().insert() - -def submit_in_background(doc: Document): - try: - doc.lock() - except frappe.DocumentLockedError: - frappe.throw( - _("This document is currently queued for execution. Please try again"), - title=_("Document Queued"), - exc=frappe.DocumentLockedError, + def after_insert(self): + job = self.queue_action( + "queue_in_background", + to_be_queued_doc=self.to_be_queued_doc, + action_for_queuing=self.action_for_queuing, ) - new_queue = frappe.new_doc("Submission Queue") - new_queue.state = "Queued" - new_queue.start_time = datetime.now() - new_queue.created_by = frappe.session.user - new_queue.ref_doctype = doc.doctype - new_queue.ref_docname = doc.name - new_queue.insert() - job = enqueue(_submit_in_background, name=new_queue.name, doc=doc) - new_queue.job_id = job.id - new_queue.save() + frappe.db.set_value(self.doctype, self.name, {"job_id": job.id}, update_modified=False) -def _submit_in_background(name: str, doc: Document): - doc.unlock() - try: - doc.submit() - frappe.db.set_value("Submission Queue", name, {"state": "Submitted"}) - except Exception as e: - frappe.db.set_value("Submission Queue", name, {"state": "Failed", "error": e}) + def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): + _action = action_for_queuing.lower() + + if _action == "update": + _action = "submit" + + try: + getattr(to_be_queued_doc, _action)() + values = {"status": "Completed"} + except Exception as e: + values = {"status": "Failed", "message": str(e)} + + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + notify(name=self.name) + + +def notify(name: str): + notification_doc = { + "type": "Notification", + "document_type": "Submission Queue", + "document_name": name, + "subject": "Job Queued", + "from_user": frappe.session.user, + "email_content": "Hello", + } + + mention = frappe.db.get_value("User", filters=frappe.db.get_value( + "Submission Queue", filters=name, fieldname="created_by" + ), fieldname="email") + if mention: + enqueue_create_notification([mention], notification_doc) + + +def queue_submission(doc: Document, action: str): + queue = frappe.new_doc("Submission Queue") + queue.state = "Queued" + queue.start_time = datetime.now() + queue.created_by = frappe.session.user + queue.ref_doctype = doc.doctype + queue.ref_docname = doc.name + queue.insert(doc, action) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 5944656a13..d6a7e70b4d 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -4,7 +4,7 @@ import json import frappe -from frappe.core.doctype.submission_queue.submission_queue import submit_in_background +from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.desk.form.load import run_onload from frappe.monitor import add_data_to_monitor @@ -19,8 +19,9 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - submit_in_background(doc) - frappe.msgprint(frappe._("Queued"), indicator="green", alert=True) + queue_submission(doc, action) + frappe.msgprint(frappe._("Queued for Submission"), indicator="green", alert=True) + return else: doc.submit() else: @@ -31,8 +32,7 @@ def savedocs(doc, action): send_updated_docs(doc) add_data_to_monitor(doctype=doc.doctype, action=action) - if not doc.meta.submit_in_background: - frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) + frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) @frappe.whitelist() From 1230b78337c06a6f539e306989de2ff822f8821c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 15:50:48 +0530 Subject: [PATCH 016/167] fix: locking the correct doctype --- .../submission_queue/submission_queue.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 621de6357b..1547177dc7 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -15,6 +15,25 @@ class SubmissionQueue(Document): self.action_for_queuing = action super().insert() + def queue_action(self, action, **kwargs): + from frappe.utils.background_jobs import enqueue + + try: + self.to_be_queued_doc.lock() + except frappe.DocumentLockedError: + frappe.throw( + _("Docuement is already queued for execution"), + title=_("Documenet Queued"), + exc=frappe.DocumentLockedError + ) + return enqueue( + "frappe.model.document.execute_action", + __doctype=self.to_be_queued_doc, + __name=self.to_be_queued_doc.name, + __action=action, + **kwargs + ) + def after_insert(self): job = self.queue_action( "queue_in_background", From b15e07dd791b326cfef81d77616d9c3776747e75 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 15:58:49 +0530 Subject: [PATCH 017/167] refactor: better locking and unlocking --- .../submission_queue/submission_queue.py | 21 ++++--------------- frappe/model/document.py | 1 - 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 1547177dc7..cb3af28a64 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -15,24 +15,11 @@ class SubmissionQueue(Document): self.action_for_queuing = action super().insert() - def queue_action(self, action, **kwargs): - from frappe.utils.background_jobs import enqueue + def lock(self, timeout=None): + self.to_be_queued_doc.lock() - try: - self.to_be_queued_doc.lock() - except frappe.DocumentLockedError: - frappe.throw( - _("Docuement is already queued for execution"), - title=_("Documenet Queued"), - exc=frappe.DocumentLockedError - ) - return enqueue( - "frappe.model.document.execute_action", - __doctype=self.to_be_queued_doc, - __name=self.to_be_queued_doc.name, - __action=action, - **kwargs - ) + def unlock(self): + frappe.get_doc(self.ref_doctype, self.ref_docname).unlock() def after_insert(self): job = self.queue_action( diff --git a/frappe/model/document.py b/frappe/model/document.py index ca27a830d3..aa55eac30a 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -3,7 +3,6 @@ import hashlib import json import time -from datetime import datetime from werkzeug.exceptions import NotFound From d4a0ad436c79b25769ca5c08f7d8a7b6802dc907 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 16:10:25 +0530 Subject: [PATCH 018/167] feat: Added completed_at field and better naming --- .../submission_queue/submission_queue.json | 39 +++++++++++-------- .../submission_queue/submission_queue.py | 7 ++-- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index f2972c5927..2bea4ada4e 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -7,28 +7,17 @@ "engine": "InnoDB", "field_order": [ "status", - "start_time", - "created_by", + "enqueued_at", + "enqueued_by", "job_id", "column_break_5", "ref_doctype", "ref_docname", "section_break_8", - "message" + "message", + "completed_at" ], "fields": [ - { - "fieldname": "start_time", - "fieldtype": "Datetime", - "label": "Start time", - "read_only": 1 - }, - { - "fieldname": "created_by", - "fieldtype": "Data", - "label": "Created By", - "read_only": 1 - }, { "fieldname": "job_id", "fieldtype": "Data", @@ -72,11 +61,29 @@ { "fieldname": "section_break_8", "fieldtype": "Section Break" + }, + { + "fieldname": "enqueued_at", + "fieldtype": "Datetime", + "label": "Enqueued At", + "read_only": 1 + }, + { + "fieldname": "enqueued_by", + "fieldtype": "Data", + "label": "Enqueued By", + "read_only": 1 + }, + { + "fieldname": "completed_at", + "fieldtype": "Data", + "label": "Completed At", + "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-04 18:41:08.495515", + "modified": "2022-10-05 16:09:36.663459", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index cb3af28a64..417657fd77 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -38,7 +38,7 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - values = {"status": "Completed"} + values = {"status": "Completed", "completed_at": datetime.now()} except Exception as e: values = {"status": "Failed", "message": str(e)} @@ -47,6 +47,7 @@ class SubmissionQueue(Document): def notify(name: str): + # Todo: fix notification notification_doc = { "type": "Notification", "document_type": "Submission Queue", @@ -66,8 +67,8 @@ def notify(name: str): def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" - queue.start_time = datetime.now() - queue.created_by = frappe.session.user + queue.enqueued_at = datetime.now() + queue.enqueued_by = frappe.session.user queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) From ae24f17e4265bb7ff2a668d3eadcb45845e271fa Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 5 Oct 2022 22:40:59 +0530 Subject: [PATCH 019/167] refactor: changed field order & added depends on --- frappe/core/doctype/doctype/doctype.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index b43a9ec3ae..c583c959aa 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -23,6 +23,7 @@ "custom", "beta", "is_virtual", + "submit_in_background", "fields_section_break", "fields", "sb1", @@ -58,7 +59,6 @@ "icon", "color", "show_preview_popup", - "submit_in_background", "show_name_in_global_search", "email_settings_sb", "default_email_template", @@ -609,6 +609,7 @@ }, { "default": "0", + "depends_on": "is_submittable", "fieldname": "submit_in_background", "fieldtype": "Check", "label": "Submit in background" @@ -694,7 +695,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-10-03 16:14:46.432770", + "modified": "2022-10-05 22:39:47.594977", "modified_by": "Administrator", "module": "Core", "name": "DocType", From cb796a4e64b49ecbd4ec5e4e03414ea5d3a1ab6d Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 6 Oct 2022 01:07:23 +0530 Subject: [PATCH 020/167] refactor(minor): better notification * changed completed_at fieldname to ended_at (in submission queue doctype) * added rollback on exception in queue_in_background method --- .../submission_queue/submission_queue.json | 12 ++-- .../submission_queue/submission_queue.py | 61 +++++++++++-------- frappe/desk/form/save.py | 10 ++- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 2bea4ada4e..25d59a8d8a 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -11,11 +11,11 @@ "enqueued_by", "job_id", "column_break_5", + "ended_at", "ref_doctype", "ref_docname", "section_break_8", - "message", - "completed_at" + "message" ], "fields": [ { @@ -75,15 +75,15 @@ "read_only": 1 }, { - "fieldname": "completed_at", - "fieldtype": "Data", - "label": "Completed At", + "fieldname": "ended_at", + "fieldtype": "Datetime", + "label": "Ended At", "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-05 16:09:36.663459", + "modified": "2022-10-06 01:11:11.835985", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 417657fd77..705baadc31 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,12 +1,11 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt -from datetime import datetime - import frappe from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document +from frappe.utils import now class SubmissionQueue(Document): @@ -15,10 +14,12 @@ class SubmissionQueue(Document): self.action_for_queuing = action super().insert() - def lock(self, timeout=None): + def lock(self): self.to_be_queued_doc.lock() def unlock(self): + # NOTE: this is called in execute_action method of Document class + # where get_doc is called hence we lose the to_be_queued_doc attribute frappe.get_doc(self.ref_doctype, self.ref_docname).unlock() def after_insert(self): @@ -26,9 +27,14 @@ class SubmissionQueue(Document): "queue_in_background", to_be_queued_doc=self.to_be_queued_doc, action_for_queuing=self.action_for_queuing, + timeout=600, + ) + frappe.db.set_value( + self.doctype, + self.name, + {"job_id": job.id, "enqueued_at": now()}, + update_modified=False, ) - frappe.db.set_value(self.doctype, self.name, {"job_id": job.id}, update_modified=False) - def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() @@ -38,37 +44,44 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - values = {"status": "Completed", "completed_at": datetime.now()} - except Exception as e: - values = {"status": "Failed", "message": str(e)} + values = {"status": "Completed"} + except Exception: + values = {"status": "Failed", "message": frappe.get_traceback()} + frappe.db.rollback() + values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - notify(name=self.name) + self.notify(values["status"], action_for_queuing) + def notify(self, submission_status: str, action: str): + if submission_status == "Failed": + doctype = "Submission Queue" + docname = self.name + message = _("Submission of {0} {1} with action {2} failed") + else: + doctype = self.ref_doctype + docname = self.ref_docname + message = _("Submission of {0} {1} with action {2} completed successfully") -def notify(name: str): - # Todo: fix notification - notification_doc = { - "type": "Notification", - "document_type": "Submission Queue", - "document_name": name, - "subject": "Job Queued", - "from_user": frappe.session.user, - "email_content": "Hello", + notification_doc = { + "type": "Alert", + "document_type": doctype, + "document_name": docname, + "subject": message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ), } - mention = frappe.db.get_value("User", filters=frappe.db.get_value( - "Submission Queue", filters=name, fieldname="created_by" - ), fieldname="email") - if mention: - enqueue_create_notification([mention], notification_doc) + notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") + enqueue_create_notification([notify_to], notification_doc) def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" - queue.enqueued_at = datetime.now() queue.enqueued_by = frappe.session.user queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) + + return queue.name diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index d6a7e70b4d..c633f3610d 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -19,8 +19,14 @@ def savedocs(doc, action): doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: if doc.meta.submit_in_background: - queue_submission(doc, action) - frappe.msgprint(frappe._("Queued for Submission"), indicator="green", alert=True) + queue_name = queue_submission(doc, action) + frappe.msgprint( + frappe._("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) return else: doc.submit() From c0b3928ac154bb08fcb894acd1b9fce130d71949 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 6 Oct 2022 02:03:27 +0530 Subject: [PATCH 021/167] refactor(minor): use virtual docfield for created_at (previously enqueued_at) field --- .../submission_queue/submission_queue.json | 17 +++++++++-------- .../submission_queue/submission_queue.py | 6 +++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 25d59a8d8a..097f233fd4 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -7,7 +7,7 @@ "engine": "InnoDB", "field_order": [ "status", - "enqueued_at", + "created_at", "enqueued_by", "job_id", "column_break_5", @@ -62,12 +62,6 @@ "fieldname": "section_break_8", "fieldtype": "Section Break" }, - { - "fieldname": "enqueued_at", - "fieldtype": "Datetime", - "label": "Enqueued At", - "read_only": 1 - }, { "fieldname": "enqueued_by", "fieldtype": "Data", @@ -79,11 +73,18 @@ "fieldtype": "Datetime", "label": "Ended At", "read_only": 1 + }, + { + "fieldname": "created_at", + "fieldtype": "Datetime", + "is_virtual": 1, + "label": "Created At", + "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-06 01:11:11.835985", + "modified": "2022-10-06 01:57:19.253609", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 705baadc31..3b7f8e10ee 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -9,6 +9,10 @@ from frappe.utils import now class SubmissionQueue(Document): + @property + def created_at(self): + return self.creation + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -32,7 +36,7 @@ class SubmissionQueue(Document): frappe.db.set_value( self.doctype, self.name, - {"job_id": job.id, "enqueued_at": now()}, + {"job_id": job.id}, update_modified=False, ) From bbc8f0baeac33cc576924f7c4d388f6eed0ca997 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 6 Oct 2022 15:22:48 +0530 Subject: [PATCH 022/167] fix: only show submit in background when doc is submittable * chore: better naming --- frappe/core/doctype/doctype/doctype.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index c583c959aa..78d2a43c75 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -23,7 +23,7 @@ "custom", "beta", "is_virtual", - "submit_in_background", + "queue_in_background", "fields_section_break", "fields", "sb1", @@ -609,10 +609,10 @@ }, { "default": "0", - "depends_on": "is_submittable", - "fieldname": "submit_in_background", + "depends_on": "eval: doc.is_submittable", + "fieldname": "queue_in_background", "fieldtype": "Check", - "label": "Submit in background" + "label": "Queue in Background" } ], "icon": "fa fa-bolt", @@ -695,7 +695,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-10-05 22:39:47.594977", + "modified": "2022-10-06 15:20:12.186038", "modified_by": "Administrator", "module": "Core", "name": "DocType", From 48e39de28afbfa528f18839b0a95170a8c5d8867 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 6 Oct 2022 18:21:45 +0530 Subject: [PATCH 023/167] fix(minor): show ref_doctype and ref_docname in listview of submission queue --- frappe/core/doctype/submission_queue/submission_queue.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 097f233fd4..15cb788f18 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -21,13 +21,13 @@ { "fieldname": "job_id", "fieldtype": "Data", - "in_list_view": 1, "label": "Job Id", "read_only": 1 }, { "fieldname": "ref_doctype", "fieldtype": "Link", + "in_list_view": 1, "label": "Reference DocType", "options": "DocType", "read_only": 1 @@ -35,6 +35,7 @@ { "fieldname": "ref_docname", "fieldtype": "Dynamic Link", + "in_list_view": 1, "label": "Reference Docname", "options": "ref_doctype", "read_only": 1 @@ -84,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-06 01:57:19.253609", + "modified": "2022-10-06 18:21:01.129702", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", From c790ad51a398684fb0b083b7da473b23b8a77255 Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 7 Oct 2022 13:14:14 +0530 Subject: [PATCH 024/167] fix(minor): only queue in background for submit action --- .../doctype/submission_queue/submission_queue.py | 8 +++++++- frappe/desk/form/save.py | 16 +++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 3b7f8e10ee..f83f7473e6 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -88,4 +88,10 @@ def queue_submission(doc: Document, action: str): queue.ref_docname = doc.name queue.insert(doc, action) - return queue.name + frappe.msgprint( + frappe._("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index c633f3610d..be41cbfaee 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -7,6 +7,7 @@ import frappe from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.desk.form.load import run_onload from frappe.monitor import add_data_to_monitor +from frappe.utils.scheduler import is_scheduler_inactive @frappe.whitelist() @@ -18,18 +19,11 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if doc.meta.submit_in_background: - queue_name = queue_submission(doc, action) - frappe.msgprint( - frappe._("Queued for Submission. You can track the progress over {0}.").format( - f"here" - ), - indicator="green", - alert=True, - ) + if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): + queue_submission(doc, action) return - else: - doc.submit() + + doc.submit() else: doc.save() From 9074e3e13dc40e4486b50d08ca38677e00544f6f Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 01:36:40 +0530 Subject: [PATCH 025/167] feat: Added unlocked button for locked documents not in queue --- .../doctype/submission_queue/submission_queue.js | 9 ++++++--- .../doctype/submission_queue/submission_queue.py | 15 +++++++++++++++ frappe/model/document.py | 3 +++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index df0833c004..573de63fdb 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -1,7 +1,10 @@ // Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on("Submission Queue", { - // refresh: function(frm) { - // } +frappe.ui.form.on('Submission Queue', { + refresh: function(frm) { + frm.add_custom_button(__("Unlock"), () => { + frm.call("unlock_doc") + }) + } }); diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index f83f7473e6..3d2aae388c 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -6,6 +6,9 @@ from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.utils import now +from rq.exceptions import NoSuchJobError +from rq.job import Job +from frappe.utils.background_jobs import get_redis_conn class SubmissionQueue(Document): @@ -79,6 +82,18 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + @frappe.whitelist() + def unlock_doc(self): + if self.is_locked: + try: + Job(self.job_id, connection=get_redis_conn()) + frappe.msgprint(_("Document already exists in queue!")) + except NoSuchJobError: + self.to_be_queued_doc.unlock() + frappe.msgprint(_("Unlocked document as no such document exists in queue")) + else: + frappe.msgprint(_("Document is already unlocked")) + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") diff --git a/frappe/model/document.py b/frappe/model/document.py index aa55eac30a..f2980a7972 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -91,6 +91,7 @@ class Document(BaseDocument): self.doctype = None self.name = None self.flags = frappe._dict() + self.is_locked = False if args and args[0]: if isinstance(args[0], str): @@ -1492,12 +1493,14 @@ class Document(BaseDocument): raise frappe.DocumentLockedError file_lock.create_lock(signature) frappe.local.locked_documents.append(self) + self.is_locked = True def unlock(self): """Delete the lock file for this document""" file_lock.delete_lock(self.get_signature()) if self in frappe.local.locked_documents: frappe.local.locked_documents.remove(self) + self.is_locked = False # validation helpers def validate_from_to_dates(self, from_date_field, to_date_field): From f28506cafb031e7ae1235f9a2512df7dac811824 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 01:51:27 +0530 Subject: [PATCH 026/167] feat: Added status updates in documents --- frappe/core/doctype/submission_queue/submission_queue.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 3d2aae388c..b1a9aeca53 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -90,11 +90,17 @@ class SubmissionQueue(Document): frappe.msgprint(_("Document already exists in queue!")) except NoSuchJobError: self.to_be_queued_doc.unlock() + self.status = "Failed" + self.save() frappe.msgprint(_("Unlocked document as no such document exists in queue")) else: + # failed, completed don't know at this point + self.status = "Failed" + self.save() frappe.msgprint(_("Document is already unlocked")) + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" From 1000a10c24bd483e79cd4d2c8946662ff22d8ba0 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 03:53:06 +0530 Subject: [PATCH 027/167] fix: fixed job status after fetching job --- .../core/doctype/submission_queue/submission_queue.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b1a9aeca53..ed632ea5d3 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -86,17 +86,13 @@ class SubmissionQueue(Document): def unlock_doc(self): if self.is_locked: try: - Job(self.job_id, connection=get_redis_conn()) - frappe.msgprint(_("Document already exists in queue!")) + job = Job(self.job_id, connection=get_redis_conn()) + if not job.get_status(refresh=True): + raise NoSuchJobError except NoSuchJobError: self.to_be_queued_doc.unlock() - self.status = "Failed" - self.save() frappe.msgprint(_("Unlocked document as no such document exists in queue")) else: - # failed, completed don't know at this point - self.status = "Failed" - self.save() frappe.msgprint(_("Document is already unlocked")) From d8ff47aac2927cc8d3e659e9eb90ec5e475ea8e3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 04:47:37 +0530 Subject: [PATCH 028/167] fix: checking and unlocking the correct doc --- .../submission_queue/submission_queue.js | 14 ++++++++------ .../submission_queue/submission_queue.json | 10 +++++++++- .../submission_queue/submission_queue.py | 18 ++++++++++++------ frappe/desk/form/save.py | 1 - frappe/model/document.py | 2 +- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 573de63fdb..1f652facd8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -1,10 +1,12 @@ // Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Submission Queue', { - refresh: function(frm) { - frm.add_custom_button(__("Unlock"), () => { - frm.call("unlock_doc") - }) - } +frappe.ui.form.on("Submission Queue", { + refresh: function (frm) { + if (frm.doc.status === "Queued") { + frm.add_custom_button(__("Unlock Reference Document"), () => { + frm.call("unlock_doc"); + }); + } + }, }); diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 15cb788f18..c4a4ff4141 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -85,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-06 18:21:01.129702", + "modified": "2022-10-08 04:26:01.657818", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -116,6 +116,14 @@ { "color": "Green", "title": "Completed" + }, + { + "color": "Yellow", + "title": "Stopped" + }, + { + "color": "Red", + "title": "Canceled" } ] } \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index ed632ea5d3..26c284dea8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,13 +1,14 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from rq.exceptions import NoSuchJobError +from rq.job import Job + import frappe from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.utils import now -from rq.exceptions import NoSuchJobError -from rq.job import Job from frappe.utils.background_jobs import get_redis_conn @@ -84,19 +85,24 @@ class SubmissionQueue(Document): @frappe.whitelist() def unlock_doc(self): - if self.is_locked: + to_be_unlocked_doc = frappe.get_doc(self.ref_doctype, self.ref_docname) + if to_be_unlocked_doc.is_locked: try: job = Job(self.job_id, connection=get_redis_conn()) - if not job.get_status(refresh=True): + status = job.get_status(refresh=True) + if not status: raise NoSuchJobError + if status := status in ("failed", "canceled", "stopped"): + to_be_unlocked_doc.unlock() + self.status = status.capitalize() + self.save() except NoSuchJobError: - self.to_be_queued_doc.unlock() + to_be_unlocked_doc.unlock() frappe.msgprint(_("Unlocked document as no such document exists in queue")) else: frappe.msgprint(_("Document is already unlocked")) - def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index be41cbfaee..f43031c899 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -22,7 +22,6 @@ def savedocs(doc, action): if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): queue_submission(doc, action) return - doc.submit() else: doc.save() diff --git a/frappe/model/document.py b/frappe/model/document.py index f2980a7972..8a86ef1bb8 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1498,9 +1498,9 @@ class Document(BaseDocument): def unlock(self): """Delete the lock file for this document""" file_lock.delete_lock(self.get_signature()) + self.is_locked = False if self in frappe.local.locked_documents: frappe.local.locked_documents.remove(self) - self.is_locked = False # validation helpers def validate_from_to_dates(self, from_date_field, to_date_field): From 6099e7e76dcaa52a2ad15e30a06886111aad3d2d Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 17:37:35 +0530 Subject: [PATCH 029/167] feat: Added more statuses for document state --- .../submission_queue/submission_queue.py | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 26c284dea8..812311dcc2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -83,24 +83,37 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + def unlock_doc_and_update_status(self, to_be_unlocked_doc: Document, possible_status: tuple): + try: + if not to_be_unlocked_doc.is_locked: + frappe.msgprint(_("Document is already unlocked updating status")) + + job = Job(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + if not status: + raise NoSuchJobError + + if status == "Queued": + frappe.msgprint(_("Document in queue for execution!")) + return + + if status := status in possible_status: + to_be_unlocked_doc.unlock() + self.status = status + self.save() + frappe.msgprint(_("Document unlocked!")) + + except NoSuchJobError: + to_be_unlocked_doc.unlock() + frappe.msgprint(_("Unlocked document as no such document exists in queue")) + @frappe.whitelist() def unlock_doc(self): + possible_status = ("Failed", "Canceled", "Stopped", "Completed") to_be_unlocked_doc = frappe.get_doc(self.ref_doctype, self.ref_docname) - if to_be_unlocked_doc.is_locked: - try: - job = Job(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - if not status: - raise NoSuchJobError - if status := status in ("failed", "canceled", "stopped"): - to_be_unlocked_doc.unlock() - self.status = status.capitalize() - self.save() - except NoSuchJobError: - to_be_unlocked_doc.unlock() - frappe.msgprint(_("Unlocked document as no such document exists in queue")) - else: - frappe.msgprint(_("Document is already unlocked")) + self.unlock_doc_and_update_status( + to_be_unlocked_doc=to_be_unlocked_doc, possible_status=possible_status + ) def queue_submission(doc: Document, action: str): From eff6c4fc5d64b183a8086bb44b6f08afa4a08f01 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 17:38:12 +0530 Subject: [PATCH 030/167] fix: checking if doc is submittable before queueing --- frappe/desk/form/save.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index f43031c899..949de9e7aa 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -19,7 +19,12 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): + if ( + action == "Submit" + and doc.meta.queue_in_background + and doc.meta.is_submittable + and not is_scheduler_inactive() + ): queue_submission(doc, action) return doc.submit() From 7f21e558dedf3ea34219fce2c6237dbfcc1ac288 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 17:39:07 +0530 Subject: [PATCH 031/167] fix(minor): better naming --- .../doctype/submission_queue/submission_queue.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 812311dcc2..7e3e98839b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -83,9 +83,9 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_doc_and_update_status(self, to_be_unlocked_doc: Document, possible_status: tuple): + def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): try: - if not to_be_unlocked_doc.is_locked: + if not doc_to_be_unlocked.is_locked: frappe.msgprint(_("Document is already unlocked updating status")) job = Job(self.job_id, connection=get_redis_conn()) @@ -98,21 +98,21 @@ class SubmissionQueue(Document): return if status := status in possible_status: - to_be_unlocked_doc.unlock() + doc_to_be_unlocked.unlock() self.status = status self.save() frappe.msgprint(_("Document unlocked!")) except NoSuchJobError: - to_be_unlocked_doc.unlock() + doc_to_be_unlocked.unlock() frappe.msgprint(_("Unlocked document as no such document exists in queue")) @frappe.whitelist() def unlock_doc(self): possible_status = ("Failed", "Canceled", "Stopped", "Completed") - to_be_unlocked_doc = frappe.get_doc(self.ref_doctype, self.ref_docname) + doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) self.unlock_doc_and_update_status( - to_be_unlocked_doc=to_be_unlocked_doc, possible_status=possible_status + doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status ) From 154e3b4e104913a31517319e67ee84d3881e10f3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 23:23:42 +0530 Subject: [PATCH 032/167] refactor: added more states to document when it's queued --- frappe/core/doctype/submission_queue/submission_queue.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index c4a4ff4141..7c744363c2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -46,7 +46,7 @@ "hidden": 1, "in_list_view": 1, "label": "Status", - "options": "Queued\nCompleted\nFailed", + "options": "Queued\nFinished\nFailed\nStopped\nCanceled", "read_only": 1 }, { @@ -85,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-08 04:26:01.657818", + "modified": "2022-10-08 18:07:02.753123", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -115,7 +115,7 @@ }, { "color": "Green", - "title": "Completed" + "title": "Finished" }, { "color": "Yellow", From 9bede49fae64ea960849945ba921aaecc7540cd3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 8 Oct 2022 23:24:07 +0530 Subject: [PATCH 033/167] fix: fixed unlocking of docs from frontend --- .../submission_queue/submission_queue.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 7e3e98839b..b2cf275890 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -52,7 +52,7 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - values = {"status": "Completed"} + values = {"status": "Finished"} except Exception: values = {"status": "Failed", "message": frappe.get_traceback()} frappe.db.rollback() @@ -76,7 +76,9 @@ class SubmissionQueue(Document): "document_type": doctype, "document_name": docname, "subject": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + f"{str(self.ref_doctype)}", + frappe.bold(self.ref_docname), + frappe.bold(action) ), } @@ -85,31 +87,34 @@ class SubmissionQueue(Document): def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): try: - if not doc_to_be_unlocked.is_locked: - frappe.msgprint(_("Document is already unlocked updating status")) - job = Job(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) if not status: raise NoSuchJobError - if status == "Queued": + if not doc_to_be_unlocked.is_locked: + frappe.msgprint(_("Document is already unlocked updating status")) + + # Checking if job is queue to be executed + if status == "queued": frappe.msgprint(_("Document in queue for execution!")) return - if status := status in possible_status: + # Checking any one of the possible termination statuses + if status in possible_status: doc_to_be_unlocked.unlock() - self.status = status + self.status = status.capitalize() self.save() frappe.msgprint(_("Document unlocked!")) except NoSuchJobError: + # Need to update status doc_to_be_unlocked.unlock() frappe.msgprint(_("Unlocked document as no such document exists in queue")) @frappe.whitelist() def unlock_doc(self): - possible_status = ("Failed", "Canceled", "Stopped", "Completed") + possible_status = ("failed", "canceled", "stopped", "finished") doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) self.unlock_doc_and_update_status( doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status From 47c75fd8bb1860f8b2789d9b87307175168b0493 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 17:26:55 +0530 Subject: [PATCH 034/167] refactor: removed redundant checking of locked documents --- frappe/core/doctype/submission_queue/submission_queue.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b2cf275890..49419b5bef 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -76,9 +76,7 @@ class SubmissionQueue(Document): "document_type": doctype, "document_name": docname, "subject": message.format( - f"{str(self.ref_doctype)}", - frappe.bold(self.ref_docname), - frappe.bold(action) + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ), } @@ -92,9 +90,6 @@ class SubmissionQueue(Document): if not status: raise NoSuchJobError - if not doc_to_be_unlocked.is_locked: - frappe.msgprint(_("Document is already unlocked updating status")) - # Checking if job is queue to be executed if status == "queued": frappe.msgprint(_("Document in queue for execution!")) @@ -130,7 +125,7 @@ def queue_submission(doc: Document, action: str): queue.insert(doc, action) frappe.msgprint( - frappe._("Queued for Submission. You can track the progress over {0}.").format( + _("Queued for Submission. You can track the progress over {0}.").format( f"here" ), indicator="green", From b44ad8acb6ff7fae10c0fdd9064f0c8416c26807 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 17:37:02 +0530 Subject: [PATCH 035/167] feat: Adding queue in background to customize form as well --- frappe/custom/doctype/customize_form/customize_form.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 05989eaa00..b22c2aba7a 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -19,6 +19,7 @@ "track_views", "allow_auto_repeat", "allow_import", + "queue_in_background", "fields_section_break", "fields", "naming_section", @@ -337,6 +338,12 @@ "fieldname": "make_attachments_public", "fieldtype": "Check", "label": "Make Attachments Public by Default" + }, + { + "default": "0", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" } ], "hide_toolbar": 1, @@ -345,7 +352,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-08-24 06:57:47.966331", + "modified": "2022-10-09 17:36:03.259470", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", From f45d1b585563b4d6393181c85d6eaf0538093bd7 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 18:25:21 +0530 Subject: [PATCH 036/167] fix: fixed redirection to doctype on notification --- frappe/core/doctype/submission_queue/submission_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 49419b5bef..5c7d8f0616 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -72,7 +72,7 @@ class SubmissionQueue(Document): message = _("Submission of {0} {1} with action {2} completed successfully") notification_doc = { - "type": "Alert", + "type": "Mention", "document_type": doctype, "document_name": docname, "subject": message.format( From 0c00c34ad64d7e20c07caf5cd4b35d3c0cc060e2 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 18:44:28 +0530 Subject: [PATCH 037/167] fix: temporary document locking --- frappe/model/document.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/model/document.py b/frappe/model/document.py index 8a86ef1bb8..63b96ebc5c 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -294,6 +294,10 @@ class Document(BaseDocument): follow_document(self.doctype, self.name, frappe.session.user) return self + def check_locked_document(self): + if self.is_locked: + raise frappe.DocumentLockedError + def save(self, *args, **kwargs): """Wrapper for _save""" return self._save(*args, **kwargs) @@ -310,6 +314,8 @@ class Document(BaseDocument): if self.flags.in_print: return + self.check_locked_document() + self.flags.notifications_executed = [] if ignore_permissions is not None: @@ -988,12 +994,14 @@ class Document(BaseDocument): @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" + self.check_locked_document() self.docstatus = DocStatus.submitted() return self.save() @whitelist.__func__ def _cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves.""" + self.check_locked_document() self.docstatus = DocStatus.cancelled() return self.save() From cde5e634e4f0845ead0bdfa8f631046e2b3e8e8d Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 21:16:59 +0530 Subject: [PATCH 038/167] feat: Adding data to monitor on action --- frappe/core/doctype/submission_queue/submission_queue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 5c7d8f0616..0ddb0945a9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -8,6 +8,7 @@ import frappe from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document +from frappe.monitor import add_data_to_monitor from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn @@ -52,6 +53,7 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() + add_data_to_monitor(doctype=to_be_queued_doc.doctype, action=_action) values = {"status": "Finished"} except Exception: values = {"status": "Failed", "message": frappe.get_traceback()} From b733c82a77237a8e282a6a676f578fd0b4477111 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 21:17:22 +0530 Subject: [PATCH 039/167] feat: Added identifier for locked state of documents refactor(minor): removed is locked setter to avoid redundancy --- frappe/model/document.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 63b96ebc5c..524bbb5fe9 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -91,7 +91,6 @@ class Document(BaseDocument): self.doctype = None self.name = None self.flags = frappe._dict() - self.is_locked = False if args and args[0]: if isinstance(args[0], str): @@ -121,6 +120,10 @@ class Document(BaseDocument): # incorrect arguments. let's not proceed. raise ValueError("Illegal arguments") + @property + def is_locked(self): + return file_lock.lock_exists(self.get_signature()) + @staticmethod def whitelist(fn): """Decorator: Whitelist method to be called remotely via REST API.""" @@ -1412,7 +1415,9 @@ class Document(BaseDocument): def get_signature(self): """Returns signature (hash) for private URL.""" - return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest() + return hashlib.sha224( + get_datetime_str(self.creation or self.modified or now()).encode() + ).hexdigest() def get_document_share_key(self, expires_on=None, no_expiry=False): if no_expiry: @@ -1501,12 +1506,10 @@ class Document(BaseDocument): raise frappe.DocumentLockedError file_lock.create_lock(signature) frappe.local.locked_documents.append(self) - self.is_locked = True def unlock(self): """Delete the lock file for this document""" file_lock.delete_lock(self.get_signature()) - self.is_locked = False if self in frappe.local.locked_documents: frappe.local.locked_documents.remove(self) From 49ca10778cc74aa1878a274dc05b1f79ef5797fc Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 9 Oct 2022 23:21:48 +0530 Subject: [PATCH 040/167] feat: Adding method to clear logs --- frappe/core/doctype/submission_queue/submission_queue.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 0ddb0945a9..7f4c1c396a 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -117,6 +117,14 @@ class SubmissionQueue(Document): doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status ) + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder.functions import Now + from frappe.query_builder import Interval + + table = frappe.qb.DocType("Submission Queue") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") From 563e3c06b0db7d85dc8ec1b94f415ccfc2e1d212 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 17:12:34 +0530 Subject: [PATCH 041/167] refactor: edited unlocked document message --- frappe/core/doctype/submission_queue/submission_queue.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 7f4c1c396a..db95228463 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -86,8 +86,9 @@ class SubmissionQueue(Document): enqueue_create_notification([notify_to], notification_doc) def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): + unlocked_doc_message = "Document Unlocked" try: - job = Job(self.job_id, connection=get_redis_conn()) + job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) if not status: raise NoSuchJobError @@ -102,12 +103,12 @@ class SubmissionQueue(Document): doc_to_be_unlocked.unlock() self.status = status.capitalize() self.save() - frappe.msgprint(_("Document unlocked!")) + frappe.msgprint(_(unlocked_doc_message)) except NoSuchJobError: # Need to update status doc_to_be_unlocked.unlock() - frappe.msgprint(_("Unlocked document as no such document exists in queue")) + frappe.msgprint(_(unlocked_doc_message)) @frappe.whitelist() def unlock_doc(self): From b33efdc6341875db53621a15ab5a5a2dbd93ddea Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 17:16:07 +0530 Subject: [PATCH 042/167] refactor: moved is_submittable validation from save to submission queue --- frappe/core/doctype/submission_queue/submission_queue.py | 8 +++++++- frappe/desk/form/save.py | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index db95228463..40e50b1f4b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -74,7 +74,7 @@ class SubmissionQueue(Document): message = _("Submission of {0} {1} with action {2} completed successfully") notification_doc = { - "type": "Mention", + "type": "Alert", "document_type": doctype, "document_name": docname, "subject": message.format( @@ -128,6 +128,12 @@ class SubmissionQueue(Document): def queue_submission(doc: Document, action: str): + # Allowing only submittable doctypes to be queued + + if not doc.meta.is_submittable: + getattr(doc, action.lower())() + return + queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.enqueued_by = frappe.session.user diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 949de9e7aa..8ea5b120e6 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -22,7 +22,6 @@ def savedocs(doc, action): if ( action == "Submit" and doc.meta.queue_in_background - and doc.meta.is_submittable and not is_scheduler_inactive() ): queue_submission(doc, action) From 01ff3d8bccb441c3cde5cb45652fbc6e9a811ce8 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 20:50:20 +0530 Subject: [PATCH 043/167] fix: fixed locking and checking --- frappe/model/document.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 524bbb5fe9..8e6fe8cf84 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -298,7 +298,7 @@ class Document(BaseDocument): return self def check_locked_document(self): - if self.is_locked: + if self.creation and self.is_locked: raise frappe.DocumentLockedError def save(self, *args, **kwargs): @@ -317,8 +317,6 @@ class Document(BaseDocument): if self.flags.in_print: return - self.check_locked_document() - self.flags.notifications_executed = [] if ignore_permissions is not None: @@ -329,6 +327,7 @@ class Document(BaseDocument): if self.get("__islocal") or not self.get("name"): return self.insert() + self.check_locked_document() self.check_permission("write", "save") self.set_user_and_timestamp() @@ -997,14 +996,12 @@ class Document(BaseDocument): @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" - self.check_locked_document() self.docstatus = DocStatus.submitted() return self.save() @whitelist.__func__ def _cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves.""" - self.check_locked_document() self.docstatus = DocStatus.cancelled() return self.save() @@ -1415,9 +1412,7 @@ class Document(BaseDocument): def get_signature(self): """Returns signature (hash) for private URL.""" - return hashlib.sha224( - get_datetime_str(self.creation or self.modified or now()).encode() - ).hexdigest() + return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest() def get_document_share_key(self, expires_on=None, no_expiry=False): if no_expiry: From 2b7ea8992908299a67d28f0abe933838b8308882 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 21:24:28 +0530 Subject: [PATCH 044/167] feat: Added submittable and queue in background logic --- .../doctype/customize_form/customize_form.json | 13 ++++++++++++- .../custom/doctype/customize_form/customize_form.py | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index b22c2aba7a..0d71aff577 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -12,6 +12,7 @@ "label", "search_fields", "column_break_5", + "is_submittable", "istable", "editable_grid", "quick_entry", @@ -341,9 +342,19 @@ }, { "default": "0", + "depends_on": "eval: doc.is_submittable", "fieldname": "queue_in_background", "fieldtype": "Check", "label": "Queue in Background" + }, + { + "default": "0", + "depends_on": "eval: doc.is_submittable", + "fetch_from": "doc_type.is_submittable", + "fieldname": "is_submittable", + "fieldtype": "Check", + "label": "Is Submittable", + "read_only": 1 } ], "hide_toolbar": 1, @@ -352,7 +363,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-10-09 17:36:03.259470", + "modified": "2022-10-11 21:23:36.669135", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index cacd38397a..2a42d249fc 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -569,6 +569,7 @@ doctype_properties = { "sort_order": "Data", "default_print_format": "Data", "allow_copy": "Check", + "is_submittable": "Check", "istable": "Check", "quick_entry": "Check", "editable_grid": "Check", From a343c06102d7d6cdb9062a03af7784d7da435a26 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 21:25:45 +0530 Subject: [PATCH 045/167] refactor: better naming --- frappe/core/doctype/submission_queue/submission_queue.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 40e50b1f4b..4bc089bd93 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -85,7 +85,7 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, possible_status: tuple): + def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, termination_statues: tuple): unlocked_doc_message = "Document Unlocked" try: job = Job.fetch(self.job_id, connection=get_redis_conn()) @@ -99,7 +99,7 @@ class SubmissionQueue(Document): return # Checking any one of the possible termination statuses - if status in possible_status: + if status in termination_statues: doc_to_be_unlocked.unlock() self.status = status.capitalize() self.save() @@ -112,10 +112,10 @@ class SubmissionQueue(Document): @frappe.whitelist() def unlock_doc(self): - possible_status = ("failed", "canceled", "stopped", "finished") + termination_statues = ("failed", "canceled", "stopped", "finished") doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) self.unlock_doc_and_update_status( - doc_to_be_unlocked=doc_to_be_unlocked, possible_status=possible_status + doc_to_be_unlocked=doc_to_be_unlocked, termination_statues=termination_statues ) @staticmethod From 418f515766aea9876e1fd159124a0fa2a2a24f7b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 11 Oct 2022 21:29:01 +0530 Subject: [PATCH 046/167] feat: Added auto log clearing --- frappe/core/doctype/log_settings/log_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 4a519dcaf4..f1b5d23c4f 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -17,6 +17,7 @@ DEFAULT_LOGTYPES_RETENTION = { "Error Snapshot": 30, "Scheduled Job Log": 90, "Route History": 90, + "Submission Queue": 30, } @@ -151,6 +152,7 @@ LOG_DOCTYPES = [ "Email Queue Recipient", "Error Snapshot", "Error Log", + "Submission Queue", ] From 9681d1965845609ccc8f2631f796245bd9b56de4 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 12 Oct 2022 23:58:19 +0530 Subject: [PATCH 047/167] feat: introduced file for tracking queues --- .../submission_queue/submission_queue.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4bc089bd93..4e46407732 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -11,6 +11,13 @@ from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn +from rq import get_current_job + + +class SubmissionQueueEntries(dict): + def save(self): + with open("./submission_queue_entries.json", "w+") as f: + f.write(frappe.json.dumps(self, indent=4)) class SubmissionQueue(Document): @@ -18,6 +25,10 @@ class SubmissionQueue(Document): def created_at(self): return self.creation + def __init__(self, *args, **kwargs): + self.queue_entries = read_submission_queue_entries() + super().__init__(*args, **kwargs) + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -38,6 +49,13 @@ class SubmissionQueue(Document): action_for_queuing=self.action_for_queuing, timeout=600, ) + self.queue_entries[job.id] = { + "DocType": self.ref_doctype, + "Docname": self.ref_docname, + "Status": job.get_status(refresh=True) + } + self.queue_entries.save() + frappe.db.set_value( self.doctype, self.name, @@ -47,6 +65,7 @@ class SubmissionQueue(Document): def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() + job = get_current_job(connection=get_redis_conn()) if _action == "update": _action = "submit" @@ -61,6 +80,13 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + self.queue_entries[job.id] = { + "DocType": to_be_queued_doc.doctype, + "Docname": to_be_queued_doc.name, + "Status": values["status"] + } + self.queue_entries.save() + self.notify(values["status"], action_for_queuing) def notify(self, submission_status: str, action: str): @@ -86,6 +112,9 @@ class SubmissionQueue(Document): enqueue_create_notification([notify_to], notification_doc) def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, termination_statues: tuple): + # Problem: If someone tries to unlock a previously failed job, + # however someone else has already queued that document again + # this will cause the queued document to be unlocked. unlocked_doc_message = "Document Unlocked" try: job = Job.fetch(self.job_id, connection=get_redis_conn()) @@ -120,13 +149,23 @@ class SubmissionQueue(Document): @staticmethod def clear_old_logs(days=30): - from frappe.query_builder.functions import Now from frappe.query_builder import Interval + from frappe.query_builder.functions import Now table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) +def read_submission_queue_entries(): + try: + with open("./submission_queue_entries.json") as f: + return SubmissionQueueEntries(frappe.json.loads(f.read())) + except FileNotFoundError: + with open("./submission_queue_entries.json", "w+") as f: + f.write(frappe.json.dumps({})) + return SubmissionQueueEntries() + + def queue_submission(doc: Document, action: str): # Allowing only submittable doctypes to be queued From 3cae3d057cf7cd387a300b2d10681221d3c1e8ac Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 13 Oct 2022 14:07:11 +0530 Subject: [PATCH 048/167] refactor(minor): made unlock_doc_and_update_status into a simple function * renamed unlock_doc_and_update_status -> unlock_reference_doc * added queued_doc property * renamed check_locked_document -> check_if_locked * reduced the statuses in submission queue * refactored unlock_reference_doc a bit --- .../submission_queue/submission_queue.json | 4 +- .../submission_queue/submission_queue.py | 84 +++++++++---------- frappe/model/document.py | 4 +- 3 files changed, 44 insertions(+), 48 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 7c744363c2..42f26a3dd2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -46,7 +46,7 @@ "hidden": 1, "in_list_view": 1, "label": "Status", - "options": "Queued\nFinished\nFailed\nStopped\nCanceled", + "options": "Queued\nFinished\nFailed", "read_only": 1 }, { @@ -85,7 +85,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-08 18:07:02.753123", + "modified": "2022-10-13 18:07:02.753123", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4e46407732..41f34bb71b 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,6 +1,7 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from rq import get_current_job from rq.exceptions import NoSuchJobError from rq.job import Job @@ -11,7 +12,6 @@ from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn -from rq import get_current_job class SubmissionQueueEntries(dict): @@ -25,6 +25,10 @@ class SubmissionQueue(Document): def created_at(self): return self.creation + @property + def queued_doc(self): + return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + def __init__(self, *args, **kwargs): self.queue_entries = read_submission_queue_entries() super().__init__(*args, **kwargs) @@ -35,24 +39,22 @@ class SubmissionQueue(Document): super().insert() def lock(self): - self.to_be_queued_doc.lock() + self.queued_doc.lock() def unlock(self): - # NOTE: this is called in execute_action method of Document class - # where get_doc is called hence we lose the to_be_queued_doc attribute - frappe.get_doc(self.ref_doctype, self.ref_docname).unlock() + self.queued_doc.unlock() def after_insert(self): job = self.queue_action( - "queue_in_background", - to_be_queued_doc=self.to_be_queued_doc, + "queue", + to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, timeout=600, ) self.queue_entries[job.id] = { "DocType": self.ref_doctype, "Docname": self.ref_docname, - "Status": job.get_status(refresh=True) + "Status": job.get_status(refresh=True), } self.queue_entries.save() @@ -63,7 +65,7 @@ class SubmissionQueue(Document): update_modified=False, ) - def queue_in_background(self, to_be_queued_doc: Document, action_for_queuing: str): + def queue(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() job = get_current_job(connection=get_redis_conn()) @@ -83,7 +85,7 @@ class SubmissionQueue(Document): self.queue_entries[job.id] = { "DocType": to_be_queued_doc.doctype, "Docname": to_be_queued_doc.name, - "Status": values["status"] + "Status": values["status"], } self.queue_entries.save() @@ -91,7 +93,7 @@ class SubmissionQueue(Document): def notify(self, submission_status: str, action: str): if submission_status == "Failed": - doctype = "Submission Queue" + doctype = self.doctype docname = self.name message = _("Submission of {0} {1} with action {2} failed") else: @@ -111,41 +113,12 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_doc_and_update_status(self, doc_to_be_unlocked: Document, termination_statues: tuple): - # Problem: If someone tries to unlock a previously failed job, - # however someone else has already queued that document again - # this will cause the queued document to be unlocked. - unlocked_doc_message = "Document Unlocked" - try: - job = Job.fetch(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - if not status: - raise NoSuchJobError - - # Checking if job is queue to be executed - if status == "queued": - frappe.msgprint(_("Document in queue for execution!")) - return - - # Checking any one of the possible termination statuses - if status in termination_statues: - doc_to_be_unlocked.unlock() - self.status = status.capitalize() - self.save() - frappe.msgprint(_(unlocked_doc_message)) - - except NoSuchJobError: - # Need to update status - doc_to_be_unlocked.unlock() - frappe.msgprint(_(unlocked_doc_message)) - @frappe.whitelist() def unlock_doc(self): - termination_statues = ("failed", "canceled", "stopped", "finished") - doc_to_be_unlocked = frappe.get_doc(self.ref_doctype, self.ref_docname) - self.unlock_doc_and_update_status( - doc_to_be_unlocked=doc_to_be_unlocked, termination_statues=termination_statues - ) + if self.status != "Queued": + return + + unlock_reference_doc(self.queued_doc, self.job_id, self.name) @staticmethod def clear_old_logs(days=30): @@ -166,6 +139,29 @@ def read_submission_queue_entries(): return SubmissionQueueEntries() +def unlock_reference_doc(ref_doc: Document, job_id: str, submission_name: str): + # TODO: If someone tries to unlock a previously failed job, + # and someone else has already queued that document again + # this will cause the queued document to be unlocked. + + try: + job = Job.fetch(job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + except NoSuchJobError: + # assuming the job failed here (?) + status = "failed" + + # Checking if job is queue to be executed/executing + if status in ("queued", "started"): + frappe.msgprint(_("Document in queue for execution!")) + + # Checking any one of the possible termination statuses + elif status in ("failed", "canceled", "stopped"): + ref_doc.unlock() + frappe.db.set_value("Submission Queue", submission_name, "status", "Failed") + frappe.msgprint(_("Document Unlocked")) + + def queue_submission(doc: Document, action: str): # Allowing only submittable doctypes to be queued diff --git a/frappe/model/document.py b/frappe/model/document.py index 8d503e5235..99e51765af 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -299,7 +299,7 @@ class Document(BaseDocument): follow_document(self.doctype, self.name, frappe.session.user) return self - def check_locked_document(self): + def check_if_locked(self): if self.creation and self.is_locked: raise frappe.DocumentLockedError @@ -329,7 +329,7 @@ class Document(BaseDocument): if self.get("__islocal") or not self.get("name"): return self.insert() - self.check_locked_document() + self.check_if_locked() self.check_permission("write", "save") self.set_user_and_timestamp() From 2547cd2d51fd6ef103b3fe28a8c79d5409699dc3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 13 Oct 2022 15:26:40 +0530 Subject: [PATCH 049/167] feat: realtime alerts if user is logged in --- .../submission_queue/submission_queue.js | 10 ++++++ .../submission_queue/submission_queue.py | 31 +++++++++++++------ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 1f652facd8..4ec0ccfde3 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -2,6 +2,16 @@ // For license information, please see license.txt frappe.ui.form.on("Submission Queue", { + setup: frappe.realtime.on("termination_status", (data) => { + let color = "green"; + if (data.status == "Failed") { + color = "orange"; + } + frappe.show_alert({ + message: data.message, + indicator: color, + }); + }), refresh: function (frm) { if (frm.doc.status === "Queued") { frm.add_custom_button(__("Unlock Reference Document"), () => { diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 41f34bb71b..bc95506f26 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -101,17 +101,28 @@ class SubmissionQueue(Document): docname = self.ref_docname message = _("Submission of {0} {1} with action {2} completed successfully") - notification_doc = { - "type": "Alert", - "document_type": doctype, - "document_name": docname, - "subject": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ), - } + if self.enqueued_by == frappe.session.user: + frappe.publish_realtime( + "termination_status", + { + "message": message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ), + "status": submission_status, + }, + ) + else: + notification_doc = { + "type": "Alert", + "document_type": doctype, + "document_name": docname, + "subject": message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ), + } - notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") - enqueue_create_notification([notify_to], notification_doc) + notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") + enqueue_create_notification([notify_to], notification_doc) @frappe.whitelist() def unlock_doc(self): From f4683cc718bc07947afbede72c0da80766db2228 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 13 Oct 2022 16:12:56 +0530 Subject: [PATCH 050/167] chore: remove unnecessary condition --- .../core/doctype/submission_queue/submission_queue.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index bc95506f26..b6b17b7cee 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -169,17 +169,13 @@ def unlock_reference_doc(ref_doc: Document, job_id: str, submission_name: str): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): ref_doc.unlock() - frappe.db.set_value("Submission Queue", submission_name, "status", "Failed") + frappe.db.set_value( + "Submission Queue", submission_name, "status", "Failed", update_modified=False + ) frappe.msgprint(_("Document Unlocked")) def queue_submission(doc: Document, action: str): - # Allowing only submittable doctypes to be queued - - if not doc.meta.is_submittable: - getattr(doc, action.lower())() - return - queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.enqueued_by = frappe.session.user From 6ea2e226f2055f595b6dbe0def096457954e33b0 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 13 Oct 2022 16:30:06 +0530 Subject: [PATCH 051/167] feat: Redirecting to doctype on alert --- frappe/core/doctype/submission_queue/submission_queue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index bc95506f26..c5f681b997 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -107,7 +107,8 @@ class SubmissionQueue(Document): { "message": message.format( frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ), + ) + + f" view it here", "status": submission_status, }, ) From 6a1f8645c3e1fbde9adbc4f25a6fc3bb318e0962 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 13 Oct 2022 17:11:49 +0530 Subject: [PATCH 052/167] feat: clearning submission queue entries & additional check for unlocking documents --- .../submission_queue/submission_queue.py | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index faa0e6bc36..5dedb6d794 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -125,12 +125,40 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + def unlock_reference_doc(self): + try: + job = Job.fetch(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + except NoSuchJobError: + # assuming the job failed here (?) + status = "failed" + + for docs in self.queue_entries.values(): + if ( + docs["DocType"] == self.ref_doctype + and docs["Docname"] == self.ref_docname + and docs["Status"] == "Queued" + ): + status = "queued" + break + # Checking if job is queue to be executed/executing + if status in ("queued", "started"): + frappe.msgprint(_("Document in queue for execution!")) + + # Checking any one of the possible termination statuses + elif status in ("failed", "canceled", "stopped"): + self.queued_doc.unlock() + frappe.db.set_value( + "Submission Queue", self.name, "status", "Failed", update_modified=False + ) + frappe.msgprint(_("Document Unlocked")) + @frappe.whitelist() def unlock_doc(self): if self.status != "Queued": return - unlock_reference_doc(self.queued_doc, self.job_id, self.name) + self.unlock_reference_doc() @staticmethod def clear_old_logs(days=30): @@ -139,6 +167,7 @@ class SubmissionQueue(Document): table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + SubmissionQueueEntries().save() def read_submission_queue_entries(): @@ -151,31 +180,6 @@ def read_submission_queue_entries(): return SubmissionQueueEntries() -def unlock_reference_doc(ref_doc: Document, job_id: str, submission_name: str): - # TODO: If someone tries to unlock a previously failed job, - # and someone else has already queued that document again - # this will cause the queued document to be unlocked. - - try: - job = Job.fetch(job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - except NoSuchJobError: - # assuming the job failed here (?) - status = "failed" - - # Checking if job is queue to be executed/executing - if status in ("queued", "started"): - frappe.msgprint(_("Document in queue for execution!")) - - # Checking any one of the possible termination statuses - elif status in ("failed", "canceled", "stopped"): - ref_doc.unlock() - frappe.db.set_value( - "Submission Queue", submission_name, "status", "Failed", update_modified=False - ) - frappe.msgprint(_("Document Unlocked")) - - def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" From 2083f9f17b4014a450c02da31044a32c74bf6561 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 14 Oct 2022 18:00:32 +0530 Subject: [PATCH 053/167] refactor: removed state management via json file for queues --- .../submission_queue/submission_queue.py | 44 +------------------ 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 5dedb6d794..1bd730a727 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -14,12 +14,6 @@ from frappe.utils import now from frappe.utils.background_jobs import get_redis_conn -class SubmissionQueueEntries(dict): - def save(self): - with open("./submission_queue_entries.json", "w+") as f: - f.write(frappe.json.dumps(self, indent=4)) - - class SubmissionQueue(Document): @property def created_at(self): @@ -30,7 +24,6 @@ class SubmissionQueue(Document): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) def __init__(self, *args, **kwargs): - self.queue_entries = read_submission_queue_entries() super().__init__(*args, **kwargs) def insert(self, to_be_queued_doc: Document, action: str): @@ -51,13 +44,6 @@ class SubmissionQueue(Document): action_for_queuing=self.action_for_queuing, timeout=600, ) - self.queue_entries[job.id] = { - "DocType": self.ref_doctype, - "Docname": self.ref_docname, - "Status": job.get_status(refresh=True), - } - self.queue_entries.save() - frappe.db.set_value( self.doctype, self.name, @@ -82,13 +68,6 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - self.queue_entries[job.id] = { - "DocType": to_be_queued_doc.doctype, - "Docname": to_be_queued_doc.name, - "Status": values["status"], - } - self.queue_entries.save() - self.notify(values["status"], action_for_queuing) def notify(self, submission_status: str, action: str): @@ -133,14 +112,6 @@ class SubmissionQueue(Document): # assuming the job failed here (?) status = "failed" - for docs in self.queue_entries.values(): - if ( - docs["DocType"] == self.ref_doctype - and docs["Docname"] == self.ref_docname - and docs["Status"] == "Queued" - ): - status = "queued" - break # Checking if job is queue to be executed/executing if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) @@ -148,9 +119,7 @@ class SubmissionQueue(Document): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): self.queued_doc.unlock() - frappe.db.set_value( - "Submission Queue", self.name, "status", "Failed", update_modified=False - ) + frappe.db.set_value("Submission Queue", self.name, "status", "Failed", update_modified=False) frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() @@ -167,17 +136,6 @@ class SubmissionQueue(Document): table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) - SubmissionQueueEntries().save() - - -def read_submission_queue_entries(): - try: - with open("./submission_queue_entries.json") as f: - return SubmissionQueueEntries(frappe.json.loads(f.read())) - except FileNotFoundError: - with open("./submission_queue_entries.json", "w+") as f: - f.write(frappe.json.dumps({})) - return SubmissionQueueEntries() def queue_submission(doc: Document, action: str): From eebbccbdbe28bed1b3dfac1d1edc4fbe46b19bda Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 14 Oct 2022 18:03:14 +0530 Subject: [PATCH 054/167] fix: added ignore permissions flag for doctype creations by other users --- frappe/core/doctype/submission_queue/submission_queue.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 1bd730a727..8870b64dc9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -23,13 +23,10 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action - super().insert() + super().insert(ignore_permissions=True) def lock(self): self.queued_doc.lock() From ae3c547e9e51cd8939089ac88970988b2fee0790 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 00:52:16 +0530 Subject: [PATCH 055/167] test: checking persistant locks on document instances --- frappe/tests/test_document_locks.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/frappe/tests/test_document_locks.py b/frappe/tests/test_document_locks.py index a92c9ffc54..2ba606685e 100644 --- a/frappe/tests/test_document_locks.py +++ b/frappe/tests/test_document_locks.py @@ -16,3 +16,24 @@ class TestDocumentLocks(FrappeTestCase): todo_1.lock() self.assertRaises(frappe.DocumentLockedError, todo.lock) todo_1.unlock() + + def test_operations_on_locked_documents(self): + todo = frappe.get_doc(dict(doctype="ToDo", description="testing operations")).insert() + todo.lock() + + with self.assertRaises(frappe.DocumentLockedError): + todo.description = "Random" + todo.save() + + # Checking for persistant locks across all instances. + doc = frappe.get_doc("ToDo", todo.name) + self.assertEquals(doc.is_locked, True) + + with self.assertRaises(frappe.DocumentLockedError): + doc.description = "Random" + doc.save() + + doc.unlock() + self.assertEquals(doc.is_locked, False) + self.assertEquals(todo.is_locked, False) + From 8bb171932d4926c1a2dca0b925fe275f92acc2ca Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 11:57:48 +0530 Subject: [PATCH 056/167] refactor: removed unnecesary actions from queue --- frappe/core/doctype/submission_queue/submission_queue.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 8870b64dc9..e0e7f8f017 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,7 +1,6 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt -from rq import get_current_job from rq.exceptions import NoSuchJobError from rq.job import Job @@ -50,7 +49,6 @@ class SubmissionQueue(Document): def queue(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() - job = get_current_job(connection=get_redis_conn()) if _action == "update": _action = "submit" From 3a10cf4977fe4ec97f276cb43768e7738c9436b9 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 13:39:32 +0530 Subject: [PATCH 057/167] fix: fixed realtime updates --- .../submission_queue/submission_queue.js | 10 --------- .../submission_queue/submission_queue.py | 22 +++++++++++++------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 4ec0ccfde3..1f652facd8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -2,16 +2,6 @@ // For license information, please see license.txt frappe.ui.form.on("Submission Queue", { - setup: frappe.realtime.on("termination_status", (data) => { - let color = "green"; - if (data.status == "Failed") { - color = "orange"; - } - frappe.show_alert({ - message: data.message, - indicator: color, - }); - }), refresh: function (frm) { if (frm.doc.status === "Queued") { frm.add_custom_button(__("Unlock Reference Document"), () => { diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index e0e7f8f017..6c40cf4814 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -22,6 +22,10 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + @property + def enqueued_by(self): + return self.owner + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -75,15 +79,20 @@ class SubmissionQueue(Document): docname = self.ref_docname message = _("Submission of {0} {1} with action {2} completed successfully") + message = ( + message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) + ) + + f" view it here" + ) + if self.enqueued_by == frappe.session.user: frappe.publish_realtime( - "termination_status", + "msgprint", { - "message": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ) - + f" view it here", - "status": submission_status, + "message": message, + "alert": True, + "indicator": "orange" if submission_status == "Failed" else "green", }, ) else: @@ -136,7 +145,6 @@ class SubmissionQueue(Document): def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" - queue.enqueued_by = frappe.session.user queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) From 3852eaea74b89dc4a0a92fe06a85550044f8aeac Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 14:41:08 +0530 Subject: [PATCH 058/167] feat: Added on_success and on_failure to enqueue --- frappe/utils/background_jobs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index e88cd75efb..98419c321f 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -53,6 +53,8 @@ def enqueue( method, queue="default", timeout=None, + on_success=None, + on_failure=None, event=None, is_async=True, job_name=None, @@ -117,6 +119,8 @@ def enqueue( return q.enqueue_call( execute_job, + on_success=on_success, + on_failure=on_failure, timeout=timeout, kwargs=queue_args, at_front=at_front, From d39c917284da1196464332a26fccc8ad3b17bd8c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 18:47:51 +0530 Subject: [PATCH 059/167] test: Added test for asserting queue action --- .../doctype/submission_queue/submission_queue.py | 4 ++++ .../submission_queue/test_submission_queue.py | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 6c40cf4814..50622b1b91 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -116,6 +116,10 @@ class SubmissionQueue(Document): # assuming the job failed here (?) status = "failed" + # Job finished successfully however action was never completed (?) + if status == "finished" and self.queued_doc.docstatus != 1: + status = "failed" + # Checking if job is queue to be executed/executing if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index d7547983a2..a9cdebdcff 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,9 +1,19 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt -# import frappe +import frappe from frappe.tests.utils import FrappeTestCase +from frappe.utils.background_jobs import get_queue class TestSubmissionQueue(FrappeTestCase): - pass + queue = get_queue(qtype="default") + + def test_queue_creation(self): + from frappe.core.doctype.submission_queue.submission_queue import queue_submission + + doc = frappe.get_doc({"doctype": "ToDo", "description": "Something"}).insert() + queue_submission(doc, "submit") + submission_queue = frappe.get_last_doc("Submission Queue") + job = self.queue.fetch_job(submission_queue.job_id) + self.assertEqual(job.get_status(refresh=True), "queued") From 0eb2458278f3bb55219648ff7e537706503ad284 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 15 Oct 2022 21:49:33 +0530 Subject: [PATCH 060/167] test: fixed queueing test and added completion check --- .../submission_queue/test_submission_queue.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index a9cdebdcff..ad19b3e592 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,6 +1,8 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt +import time + import frappe from frappe.tests.utils import FrappeTestCase from frappe.utils.background_jobs import get_queue @@ -9,11 +11,18 @@ from frappe.utils.background_jobs import get_queue class TestSubmissionQueue(FrappeTestCase): queue = get_queue(qtype="default") - def test_queue_creation(self): + def test_queue_operation(self): from frappe.core.doctype.submission_queue.submission_queue import queue_submission doc = frappe.get_doc({"doctype": "ToDo", "description": "Something"}).insert() queue_submission(doc, "submit") submission_queue = frappe.get_last_doc("Submission Queue") + + # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) - self.assertEqual(job.get_status(refresh=True), "queued") + self.assertIn(job.get_status(refresh=True), ("queued", "started")) + + time.sleep(2) + + # Test completion + self.assertEqual(job.get_status(refresh=True), "finished") From 2172d5f40605128f6bcf253bbb3ddb1f8cdadf90 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sun, 16 Oct 2022 18:32:36 +0530 Subject: [PATCH 061/167] test: removed completion check --- .../core/doctype/submission_queue/test_submission_queue.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index ad19b3e592..3ec99f06b2 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,8 +1,6 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt -import time - import frappe from frappe.tests.utils import FrappeTestCase from frappe.utils.background_jobs import get_queue @@ -21,8 +19,3 @@ class TestSubmissionQueue(FrappeTestCase): # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) self.assertIn(job.get_status(refresh=True), ("queued", "started")) - - time.sleep(2) - - # Test completion - self.assertEqual(job.get_status(refresh=True), "finished") From 48f20807692d0235e2aaa5602bbaf5d34048010a Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 20 Oct 2022 22:35:08 +0530 Subject: [PATCH 062/167] refactor: better function naming --- frappe/database/query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 0cda2fdb43..2e2185944f 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -287,7 +287,7 @@ class Engine: return conditions @staticmethod - def get_condition_from_nested_sets(value: list | tuple, table: str): + def get_nested_set_hierarchy_result(value: list | tuple, table: str): field = frappe.meta.get_field("name") ref_doctype = field.options if field else table lft, rgt = "", "" @@ -382,7 +382,7 @@ class Engine: # Nested set support if isinstance(value, (list, tuple)): if value in self.OPERATOR_MAP["nested_set"]: - result = self.get_condition_from_nested_sets(value, table) + result = self.get_nested_set_hierarchy_result(value, table) if result: _value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in result] _operator = ( From 35ce3c0ecc1f045bd01dd717a0d5837e7a185d2f Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 21 Oct 2022 16:10:22 +0530 Subject: [PATCH 063/167] fix: fixed nested query results --- frappe/database/query.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 2e2185944f..88baf8b715 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -288,11 +288,10 @@ class Engine: @staticmethod def get_nested_set_hierarchy_result(value: list | tuple, table: str): - field = frappe.meta.get_field("name") - ref_doctype = field.options if field else table + ref_doctype = table lft, rgt = "", "" lft, rgt = ( - frappe.qb.from_(ref_doctype).select(["lft", "rgt"]).where(Field("name") == value[1]).run() + frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == value[1]).run()[0] ) if value in ("descendants of", "not descendants of"): @@ -381,7 +380,7 @@ class Engine: continue # Nested set support if isinstance(value, (list, tuple)): - if value in self.OPERATOR_MAP["nested_set"]: + if value[0] in self.OPERATOR_MAP["nested_set"]: result = self.get_nested_set_hierarchy_result(value, table) if result: _value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in result] From a65db344eea72cf6f6f6f04e1e47db1d3afa9e2b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 21 Oct 2022 19:44:15 +0530 Subject: [PATCH 064/167] fix: empty nested result clause --- frappe/database/query.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 88baf8b715..7a46d64dbc 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -287,14 +287,13 @@ class Engine: return conditions @staticmethod - def get_nested_set_hierarchy_result(value: list | tuple, table: str): + def get_nested_set_hierarchy_result(hierarchy: str, field: str, table: str): ref_doctype = table lft, rgt = "", "" lft, rgt = ( - frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == value[1]).run()[0] + frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == field).run()[0] ) - - if value in ("descendants of", "not descendants of"): + if hierarchy in ("descendants of", "not descendants of"): result = ( frappe.qb.from_(ref_doctype) .select(Field("name")) @@ -381,15 +380,17 @@ class Engine: # Nested set support if isinstance(value, (list, tuple)): if value[0] in self.OPERATOR_MAP["nested_set"]: - result = self.get_nested_set_hierarchy_result(value, table) - if result: - _value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in result] - _operator = ( + hierarchy, _field = value + result = self.get_nested_set_hierarchy_result(hierarchy, _field, table) + _operator = ( self.OPERATOR_MAP["not in"] - if value in ("not ancestors of", "not descendants of") + if hierarchy in ("not ancestors of", "not descendants of") else self.OPERATOR_MAP["in"] ) - return conditions.where(_operator(getattr(table, key), _value)) + if result: + return conditions.where(_operator(getattr(table, key), result[0])) + else: + return conditions.where(_operator(getattr(table, key),("",))) _operator = self.OPERATOR_MAP[value[0].casefold()] _value = value[1] if value[1] else ("",) From 69112cec39806671a2e9729dce5f02cb17e1e99b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 25 Oct 2022 18:05:48 +0530 Subject: [PATCH 065/167] refactor: removed static methods --- frappe/database/query.py | 69 ++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 7a46d64dbc..d55d326573 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -169,6 +169,34 @@ def table_from_string(table: str) -> "DocType": return frappe.qb.DocType(table_name=table_name.replace("`", "")) +def get_nested_set_hierarchy_result(hierarchy: str, field: str, table: str): + ref_doctype = table + lft, rgt = "", "" + lft, rgt = ( + frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == field).run()[0] + ) + if hierarchy in ("descendants of", "not descendants of"): + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") > lft) + .where(Field("rgt") < rgt) + .orderby(Field("lft"), order=Order.asc) + .run() + ) + else: + # Get ancestor elements of a DocType with a tree structure + result = ( + frappe.qb.from_(ref_doctype) + .select(Field("name")) + .where(Field("lft") < lft) + .where(Field("rgt") > rgt) + .orderby(Field("lft"), order=Order.desc) + .run() + ) + return result + + # default operators OPERATOR_MAP: dict[str, Callable] = { "+": operator.add, @@ -286,34 +314,6 @@ class Engine: return conditions - @staticmethod - def get_nested_set_hierarchy_result(hierarchy: str, field: str, table: str): - ref_doctype = table - lft, rgt = "", "" - lft, rgt = ( - frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == field).run()[0] - ) - if hierarchy in ("descendants of", "not descendants of"): - result = ( - frappe.qb.from_(ref_doctype) - .select(Field("name")) - .where(Field("lft") > lft) - .where(Field("rgt") < rgt) - .orderby(Field("lft"), order=Order.asc) - .run() - ) - else: - # Get ancestor elements of a DocType with a tree structure - result = ( - frappe.qb.from_(ref_doctype) - .select(Field("name")) - .where(Field("lft") < lft) - .where(Field("rgt") > rgt) - .orderby(Field("lft"), order=Order.desc) - .run() - ) - return result - def misc_query(self, table: str, filters: list | tuple = None, **kwargs): """Build conditions using the given Lists or Tuple filters @@ -381,16 +381,16 @@ class Engine: if isinstance(value, (list, tuple)): if value[0] in self.OPERATOR_MAP["nested_set"]: hierarchy, _field = value - result = self.get_nested_set_hierarchy_result(hierarchy, _field, table) + result = get_nested_set_hierarchy_result(hierarchy, _field, table) _operator = ( - self.OPERATOR_MAP["not in"] - if hierarchy in ("not ancestors of", "not descendants of") - else self.OPERATOR_MAP["in"] - ) + self.OPERATOR_MAP["not in"] + if hierarchy in ("not ancestors of", "not descendants of") + else self.OPERATOR_MAP["in"] + ) if result: return conditions.where(_operator(getattr(table, key), result[0])) else: - return conditions.where(_operator(getattr(table, key),("",))) + return conditions.where(_operator(getattr(table, key), ("",))) _operator = self.OPERATOR_MAP[value[0].casefold()] _value = value[1] if value[1] else ("",) @@ -662,6 +662,7 @@ class Engine: has_join = True if has_join: + def _update_pypika_fields(field): if not is_pypika_function_object(field): field = field if isinstance(field, str) else field.get_sql() From e16ca035725d6b89d582798135fb02704a461d71 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 25 Oct 2022 20:34:47 +0530 Subject: [PATCH 066/167] fix: fixed multiple descendants --- frappe/database/query.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index d55d326573..063e8f9eaf 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -1,3 +1,4 @@ +import itertools import operator import re from ast import literal_eval @@ -388,7 +389,8 @@ class Engine: else self.OPERATOR_MAP["in"] ) if result: - return conditions.where(_operator(getattr(table, key), result[0])) + result = list(itertools.chain.from_iterable(result)) + return conditions.where(_operator(getattr(table, key), result)) else: return conditions.where(_operator(getattr(table, key), ("",))) From 7d18d5f519ed83c5e7314aee5fdccd821135ed01 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 25 Oct 2022 20:42:33 +0530 Subject: [PATCH 067/167] test: Added tests for nested set queries --- frappe/tests/test_query.py | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 1804f1672c..ff9df76368 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -1,8 +1,11 @@ +import itertools import frappe +from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.query_builder import Field from frappe.query_builder.functions import Abs, Count, Ifnull, Max, Now, Timestamp from frappe.tests.test_query_builder import db_type_is, run_only_if from frappe.tests.utils import FrappeTestCase +from frappe.utils.nestedset import get_descendants_of class TestQuery(FrappeTestCase): @@ -207,3 +210,54 @@ class TestQuery(FrappeTestCase): self.assertNotIn( "email", frappe.qb.engine.get_query("User", fields=["name", "#email"], filters={}).get_sql() ) + + def test_nestedset(self): + frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") + frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") + records = [ + { + "some_fieldname": "Root Node", + "parent_test_tree_doctype": None, + "is_group": 1, + }, + { + "some_fieldname": "Parent 1", + "parent_test_tree_doctype": "Root Node", + "is_group": 1, + }, + { + "some_fieldname": "Parent 2", + "parent_test_tree_doctype": "Root Node", + "is_group": 1, + }, + { + "some_fieldname": "Child 1", + "parent_test_tree_doctype": "Parent 1", + "is_group": 0, + }, + { + "some_fieldname": "Child 2", + "parent_test_tree_doctype": "Parent 1", + "is_group": 0, + }, + { + "some_fieldname": "Child 3", + "parent_test_tree_doctype": "Parent 2", + "is_group": 0, + }, + ] + + tree_doctype = new_doctype("Test Tree DocType", is_tree=True, autoname="field:some_fieldname") + tree_doctype.insert() + + for record in records: + d = frappe.new_doc("Test Tree DocType") + d.update(record) + d.insert() + + result = frappe.qb.engine.get_query( + "Test Tree DocType", fields=["name"], filters={"name": ("descendants of", "Parent 1")} + ).run(as_list=1) + result = list(itertools.chain.from_iterable(result)) + result.reverse() + self.assertListEqual(result, get_descendants_of("Test Tree DocType", "Parent 1")) \ No newline at end of file From 15db8228f4142136bd805fce31d0a56720e0f797 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 26 Oct 2022 15:58:24 +0530 Subject: [PATCH 068/167] fix: adding additional conditions --- frappe/database/query.py | 6 ++++-- frappe/tests/test_query.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 063e8f9eaf..40efa10b74 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -390,9 +390,11 @@ class Engine: ) if result: result = list(itertools.chain.from_iterable(result)) - return conditions.where(_operator(getattr(table, key), result)) + conditions = conditions.where(_operator(getattr(table, key), result)) else: - return conditions.where(_operator(getattr(table, key), ("",))) + conditions = conditions.where(_operator(getattr(table, key), ("",))) + # Allow additional conditions + break _operator = self.OPERATOR_MAP[value[0].casefold()] _value = value[1] if value[1] else ("",) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index ff9df76368..1dd142aa82 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -1,4 +1,5 @@ import itertools + import frappe from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.query_builder import Field @@ -256,8 +257,13 @@ class TestQuery(FrappeTestCase): d.insert() result = frappe.qb.engine.get_query( - "Test Tree DocType", fields=["name"], filters={"name": ("descendants of", "Parent 1")} + "Test Tree DocType", + fields=["name"], + filters={"name": ("descendants of", "Parent 1")}, + orderby="modified", ).run(as_list=1) result = list(itertools.chain.from_iterable(result)) - result.reverse() - self.assertListEqual(result, get_descendants_of("Test Tree DocType", "Parent 1")) \ No newline at end of file + self.assertListEqual(result, get_descendants_of("Test Tree DocType", "Parent 1")) + + frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") + frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") From 6198177ec617f10ca32a3f82f6e7d27b0d5441ba Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 27 Oct 2022 12:58:17 +0530 Subject: [PATCH 069/167] test: Added tests for ancestors --- frappe/tests/test_query.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 1dd142aa82..91d9cdfc9e 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -6,7 +6,7 @@ from frappe.query_builder import Field from frappe.query_builder.functions import Abs, Count, Ifnull, Max, Now, Timestamp from frappe.tests.test_query_builder import db_type_is, run_only_if from frappe.tests.utils import FrappeTestCase -from frappe.utils.nestedset import get_descendants_of +from frappe.utils.nestedset import get_ancestors_of, get_descendants_of class TestQuery(FrappeTestCase): @@ -212,9 +212,7 @@ class TestQuery(FrappeTestCase): "email", frappe.qb.engine.get_query("User", fields=["name", "#email"], filters={}).get_sql() ) - def test_nestedset(self): - frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") - frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") + def insert_tree_docs(self): records = [ { "some_fieldname": "Root Node", @@ -256,14 +254,32 @@ class TestQuery(FrappeTestCase): d.update(record) d.insert() - result = frappe.qb.engine.get_query( + + def test_nestedset(self): + frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") + frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") + self.insert_tree_docs() + descendants_result = frappe.qb.engine.get_query( "Test Tree DocType", fields=["name"], filters={"name": ("descendants of", "Parent 1")}, orderby="modified", ).run(as_list=1) - result = list(itertools.chain.from_iterable(result)) - self.assertListEqual(result, get_descendants_of("Test Tree DocType", "Parent 1")) + + # Format decendants result + descendants_result = list(itertools.chain.from_iterable(descendants_result)) + self.assertListEqual(descendants_result, get_descendants_of("Test Tree DocType", "Parent 1")) + + ancestors_result = frappe.qb.engine.get_query( + "Test Tree DocType", + fields=["name"], + filters={"name": ("ancestors of", "Child 2")}, + orderby="modified", + ).run(as_list=1) + + # Format ancestors result + ancestors_result = list(itertools.chain.from_iterable(ancestors_result)) + self.assertListEqual(ancestors_result, get_ancestors_of("Test Tree DocType", "Child 2")) frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") From e6a281f19e0583939a1a153ee6d2e71eae57faa6 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 27 Oct 2022 13:11:15 +0530 Subject: [PATCH 070/167] refactor: better naming --- frappe/database/query.py | 6 +++--- frappe/database/utils.py | 2 +- frappe/model/db_query.py | 4 ++-- frappe/tests/test_query.py | 1 - 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 40efa10b74..bbdd153afd 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -11,7 +11,7 @@ from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder import frappe from frappe import _ -from frappe.database.utils import NESTED_SET_HIERARCHY, is_pypika_function_object +from frappe.database.utils import NestedSetHierarchy, is_pypika_function_object from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table, functions from frappe.query_builder.functions import Function, SqlFunctions @@ -220,7 +220,7 @@ OPERATOR_MAP: dict[str, Callable] = { "between": func_between, "is": func_is, "timespan": func_timespan, - "nested_set": NESTED_SET_HIERARCHY, + "nested_set": NestedSetHierarchy, # TODO: Add support for custom operators (WIP) - via filters_config hooks } @@ -390,7 +390,7 @@ class Engine: ) if result: result = list(itertools.chain.from_iterable(result)) - conditions = conditions.where(_operator(getattr(table, key), result)) + conditions = conditions.where(_operator(getattr(table, key), result)) else: conditions = conditions.where(_operator(getattr(table, key), ("",))) # Allow additional conditions diff --git a/frappe/database/utils.py b/frappe/database/utils.py index 551931d755..4ea039e5a7 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -18,7 +18,7 @@ QueryValues = tuple | list | dict | NoneType EmptyQueryValues = object() FallBackDateTimeStr = "0001-01-01 00:00:00.000000" -NESTED_SET_HIERARCHY = ( +NestedSetHierarchy = ( "ancestors of", "descendants of", "not ancestors of", diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index d54f1adbbc..3e6b8ec753 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -13,7 +13,7 @@ import frappe.permissions import frappe.share from frappe import _ from frappe.core.doctype.server_script.server_script_utils import get_server_script_map -from frappe.database.utils import NESTED_SET_HIERARCHY, FallBackDateTimeStr +from frappe.database.utils import FallBackDateTimeStr, NestedSetHierarchy from frappe.model import optional_fields from frappe.model.meta import get_table_columns from frappe.model.utils.user_settings import get_user_settings, update_user_settings @@ -568,7 +568,7 @@ class DatabaseQuery: can_be_null = True # prepare in condition - if f.operator.lower() in NESTED_SET_HIERARCHY: + if f.operator.lower() in NestedSetHierarchy: values = f.value or "" # TODO: handle list and tuple diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 91d9cdfc9e..f7fff6fbf3 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -254,7 +254,6 @@ class TestQuery(FrappeTestCase): d.update(record) d.insert() - def test_nestedset(self): frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") From ff1389327a424ca8940dcde4951ef78a821f5bbd Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 18:23:20 +0530 Subject: [PATCH 071/167] chore: rename queue to background_submission --- .../doctype/submission_queue/submission_queue.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 50622b1b91..9b4411caae 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -18,14 +18,14 @@ class SubmissionQueue(Document): def created_at(self): return self.creation - @property - def queued_doc(self): - return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) - @property def enqueued_by(self): return self.owner + @property + def queued_doc(self): + return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -39,7 +39,7 @@ class SubmissionQueue(Document): def after_insert(self): job = self.queue_action( - "queue", + "background_submission", to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, timeout=600, @@ -51,7 +51,7 @@ class SubmissionQueue(Document): update_modified=False, ) - def queue(self, to_be_queued_doc: Document, action_for_queuing: str): + def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): _action = action_for_queuing.lower() if _action == "update": @@ -86,6 +86,7 @@ class SubmissionQueue(Document): + f" view it here" ) + # TODO: this is messed up if self.enqueued_by == frappe.session.user: frappe.publish_realtime( "msgprint", From 1856d2eb3f42cab4d5d88f176d2b56349092e819 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 19:11:13 +0530 Subject: [PATCH 072/167] fix: realtime alert use session object's last updated time to figure out whether to send a realtime alert or a notification --- .../submission_queue/submission_queue.py | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 9b4411caae..69523ad3cd 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,6 +1,8 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from datetime import datetime + from rq.exceptions import NoSuchJobError from rq.job import Job @@ -9,7 +11,7 @@ from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.monitor import add_data_to_monitor -from frappe.utils import now +from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds from frappe.utils.background_jobs import get_redis_conn @@ -26,9 +28,10 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) - def insert(self, to_be_queued_doc: Document, action: str): + def insert(self, to_be_queued_doc: Document, action: str, session_id: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action + self.enqueued_by_session_id = session_id super().insert(ignore_permissions=True) def lock(self): @@ -42,6 +45,7 @@ class SubmissionQueue(Document): "background_submission", to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, + enqueued_by_session_id=self.enqueued_by_session_id, timeout=600, ) frappe.db.set_value( @@ -51,7 +55,9 @@ class SubmissionQueue(Document): update_modified=False, ) - def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): + def background_submission( + self, to_be_queued_doc: Document, action_for_queuing: str, enqueued_by_session_id: str + ): _action = action_for_queuing.lower() if _action == "update": @@ -67,9 +73,9 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - self.notify(values["status"], action_for_queuing) + self.notify(values["status"], action_for_queuing, enqueued_by_session_id) - def notify(self, submission_status: str, action: str): + def notify(self, submission_status: str, action: str, session_id: str): if submission_status == "Failed": doctype = self.doctype docname = self.name @@ -79,37 +85,33 @@ class SubmissionQueue(Document): docname = self.ref_docname message = _("Submission of {0} {1} with action {2} completed successfully") - message = ( - message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ) - + f" view it here" + message = message.format( + frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - # TODO: this is messed up - if self.enqueued_by == frappe.session.user: + if time_diff_in_seconds(now(), get_user_last_request_time(session_id)) < 60: frappe.publish_realtime( "msgprint", { - "message": message, + "message": message + + f". View it here", "alert": True, - "indicator": "orange" if submission_status == "Failed" else "green", + "indicator": "red" if submission_status == "Failed" else "green", }, + user=self.enqueued_by, ) else: notification_doc = { "type": "Alert", "document_type": doctype, "document_name": docname, - "subject": message.format( - frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) - ), + "subject": message, } notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - def unlock_reference_doc(self): + def _unlock_reference_doc(self): try: job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) @@ -136,7 +138,7 @@ class SubmissionQueue(Document): if self.status != "Queued": return - self.unlock_reference_doc() + self._unlock_reference_doc() @staticmethod def clear_old_logs(days=30): @@ -147,12 +149,21 @@ class SubmissionQueue(Document): frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) +def get_user_last_request_time(session_id): + return ( + frappe.cache() + .hget("session", session_id) + .get("data", {}) + .get("last_updated", datetime.min.strftime(DATETIME_FORMAT)) + ) + + def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.ref_doctype = doc.doctype queue.ref_docname = doc.name - queue.insert(doc, action) + queue.insert(doc, action, frappe.session.sid) frappe.msgprint( _("Queued for Submission. You can track the progress over {0}.").format( From a21eb1efd6648a838d5dc6b0aff28b2c14de12a2 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 19:19:32 +0530 Subject: [PATCH 073/167] refactor(minor): move staticmethod after properties --- .../submission_queue/submission_queue.json | 3 ++- .../doctype/submission_queue/submission_queue.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 42f26a3dd2..d0579ac599 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -66,6 +66,7 @@ { "fieldname": "enqueued_by", "fieldtype": "Data", + "is_virtual": 1, "label": "Enqueued By", "read_only": 1 }, @@ -85,7 +86,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-13 18:07:02.753123", + "modified": "2022-10-30 19:16:59.284681", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 69523ad3cd..34ccfcb482 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -28,6 +28,14 @@ class SubmissionQueue(Document): def queued_doc(self): return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Submission Queue") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + def insert(self, to_be_queued_doc: Document, action: str, session_id: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action @@ -140,14 +148,6 @@ class SubmissionQueue(Document): self._unlock_reference_doc() - @staticmethod - def clear_old_logs(days=30): - from frappe.query_builder import Interval - from frappe.query_builder.functions import Now - - table = frappe.qb.DocType("Submission Queue") - frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) - def get_user_last_request_time(session_id): return ( From 2952077a6d54bda3832f69a662027ddc5329ea8b Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 19:23:15 +0530 Subject: [PATCH 074/167] refactor(minor): remove additional document states and rename message to exception --- .../submission_queue/submission_queue.json | 24 +++++++------------ .../submission_queue/submission_queue.py | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index d0579ac599..00e841b926 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -15,7 +15,7 @@ "ref_doctype", "ref_docname", "section_break_8", - "message" + "exception" ], "fields": [ { @@ -53,12 +53,6 @@ "fieldname": "column_break_5", "fieldtype": "Column Break" }, - { - "fieldname": "message", - "fieldtype": "Text", - "label": "Message", - "read_only": 1 - }, { "fieldname": "section_break_8", "fieldtype": "Section Break" @@ -82,11 +76,17 @@ "is_virtual": 1, "label": "Created At", "read_only": 1 + }, + { + "fieldname": "exception", + "fieldtype": "Text", + "label": "Exception", + "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-30 19:16:59.284681", + "modified": "2022-10-30 19:22:20.998753", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", @@ -117,14 +117,6 @@ { "color": "Green", "title": "Finished" - }, - { - "color": "Yellow", - "title": "Stopped" - }, - { - "color": "Red", - "title": "Canceled" } ] } \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 34ccfcb482..e7269b5d37 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -76,7 +76,7 @@ class SubmissionQueue(Document): add_data_to_monitor(doctype=to_be_queued_doc.doctype, action=_action) values = {"status": "Finished"} except Exception: - values = {"status": "Failed", "message": frappe.get_traceback()} + values = {"status": "Failed", "exception": frappe.get_traceback()} frappe.db.rollback() values["ended_at"] = now() From 759ee1a1164901abc329aed10f28d11e89a2500c Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 23:16:09 +0530 Subject: [PATCH 075/167] feat(minor): submission queue banner for showing recent submissions --- .../submission_queue/submission_queue.py | 13 ++++ frappe/public/js/frappe/form/form.js | 63 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index e7269b5d37..37d07f3239 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -172,3 +172,16 @@ def queue_submission(doc: Document, action: str): indicator="green", alert=True, ) + + +@frappe.whitelist() +def get_latest_submissions(doctype, docname): + # NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere + # hence assuming modified will be equal to creation for submission queue documents + + dt = "Submission Queue" + filters = {"ref_doctype": doctype, "ref_docname": docname} + return { + "latest_submission": frappe.db.get_value(dt, filters), + "latest_failed_submission": frappe.db.get_value(dt, filters.update({"status": "Failed"})), + } diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8c642a73f0..6999edbea2 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -450,6 +450,7 @@ frappe.ui.form.Form = class FrappeForm { .toggleClass("cancelled-form", this.doc.docstatus === 2); this.show_conflict_message(); + this.show_submission_queue_banner(); if (frappe.boot.read_only) { this.disable_form(); @@ -2036,6 +2037,68 @@ frappe.ui.form.Form = class FrappeForm { .filter((user) => !["Administrator", frappe.session.user].includes(user)) .filter(Boolean); } + + show_submission_queue_banner() { + if ( + !( + this.meta.is_submittable && + this.meta.queue_in_background && + !this.doc.__islocal && + this.doc.docstatus === 0 + ) + ) + return; + + let wrapper = this.layout.wrapper.find(".submission-queue-banner"); + if (!wrapper.length) { + wrapper = $('
'); + this.layout.wrapper.prepend(wrapper); + } + + frappe + .call({ + method: "frappe.core.doctype.submission_queue.submission_queue.get_latest_submissions", + args: { doctype: this.doctype, docname: this.docname }, + }) + .then((r) => { + if (r.message.latest_submission) { + // if we are here that means some submission(s) were queued and are in queued/failed state + wrapper.show(); + let col_width = 4; + let failed_link = ""; + if (r.message.latest_failed_submission) { + col_width = 3; + failed_link = `
+ ${__( + "Latest Failed Submission" + )} +
`; + } + + let html = ` +
+
+ ${__("Submission Status:")} +
+
+ ${__("Latest Submission")} +
+ ${failed_link} +
+ ${__( + "All Submissions" + )} +
+
+ `; + + wrapper.html(html); + } else { + wrapper.hide(); + wrapper.html(""); + } + }); + } }; frappe.validated = 0; From c3e2cd1629aeab25fdc552ebf229d942698d603c Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 30 Oct 2022 23:42:14 +0530 Subject: [PATCH 076/167] fix: only allow queue_in_background in customize form --- .../doctype/customize_form/customize_form.js | 4 ++++ .../customize_form/customize_form.json | 23 +++++-------------- .../doctype/customize_form/customize_form.py | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 759a9e1b3a..6f94f32256 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -149,6 +149,10 @@ frappe.ui.form.on("Customize Form", { const is_autoname_autoincrement = frm.doc.autoname === "autoincrement"; frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); + frm.toggle_display( + ["queue_in_background"], + frappe.get_meta(frm.doc.doc_type).is_submittable || 0 + ); } frm.events.setup_export(frm); diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index a64d1fa05a..b9fb52d1dc 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -12,7 +12,6 @@ "label", "search_fields", "column_break_5", - "is_submittable", "istable", "is_calendar_and_gantt", "editable_grid", @@ -344,21 +343,11 @@ "label": "Make Attachments Public by Default" }, { - "default": "0", - "depends_on": "eval: doc.is_submittable", - "fieldname": "queue_in_background", - "fieldtype": "Check", - "label": "Queue in Background" - }, - { - "default": "0", - "depends_on": "eval: doc.is_submittable", - "fetch_from": "doc_type.is_submittable", - "fieldname": "is_submittable", - "fieldtype": "Check", - "label": "Is Submittable", - "read_only": 1 - }, + "default": "0", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" + }, { "fieldname": "default_view", "fieldtype": "Select", @@ -385,7 +374,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-08-30 11:45:16.772277", + "modified": "2022-10-30 23:39:49.628093", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 920df661e3..bdd18cddfa 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -569,9 +569,9 @@ doctype_properties = { "sort_order": "Data", "default_print_format": "Data", "allow_copy": "Check", - "is_submittable": "Check", "istable": "Check", "quick_entry": "Check", + "queue_in_background": "Check", "editable_grid": "Check", "max_attachments": "Int", "make_attachments_public": "Check", From b0f3016c760e6d4c68d2a6b5fd7eba837f235b5d Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 31 Oct 2022 00:52:31 +0530 Subject: [PATCH 077/167] chore: add confirm dialog for unlock doc button --- frappe/core/doctype/submission_queue/submission_queue.js | 4 +++- frappe/core/doctype/submission_queue/submission_queue.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 1f652facd8..414c8c9ee0 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -5,7 +5,9 @@ frappe.ui.form.on("Submission Queue", { refresh: function (frm) { if (frm.doc.status === "Queued") { frm.add_custom_button(__("Unlock Reference Document"), () => { - frm.call("unlock_doc"); + frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => { + frm.call("unlock_doc"); + }); }); } }, diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 37d07f3239..3bb74318f0 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -13,6 +13,7 @@ from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.data import cint class SubmissionQueue(Document): @@ -97,7 +98,7 @@ class SubmissionQueue(Document): frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - if time_diff_in_seconds(now(), get_user_last_request_time(session_id)) < 60: + if cint(time_diff_in_seconds(now(), get_user_last_request_time(session_id))) <= 60: frappe.publish_realtime( "msgprint", { From 51ae7afdee6fd4b07e4fdd373c231feaeee16c08 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 31 Oct 2022 12:29:56 +0530 Subject: [PATCH 078/167] feat: reload listener for form --- .../submission_queue/submission_queue.py | 2 ++ frappe/public/js/frappe/form/form.js | 32 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 3bb74318f0..c0748ebf58 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -120,6 +120,8 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) + frappe.publish_realtime(f"reload_doc_{self.ref_doctype}_{self.ref_docname}") + def _unlock_reference_doc(self): try: job = Job.fetch(self.job_id, connection=get_redis_conn()) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 6999edbea2..8027f32454 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -397,6 +397,8 @@ frappe.ui.form.Form = class FrappeForm { // set the doc this.doc = frappe.get_doc(this.doctype, this.docname); + if (!this.doc.__islocal) this.setup_reload_listener(); + // check permissions this.fetch_permissions(); if (!this.has_read_permission()) { @@ -1977,6 +1979,19 @@ frappe.ui.form.Form = class FrappeForm { }); } + setup_reload_listener() { + let doctype = this.doctype; + let docname = this.docname; + let listener_name = `reload_doc_${doctype}_${docname}`; + + frappe.realtime.off(listener_name); + frappe.realtime.on(listener_name, () => { + if (frappe.get_route_str() === `Form/${doctype}/${docname}`) { + this.reload_doc(); + } + }); + } + // Filters fields from the reference doctype and sets them as options for a Select field set_fields_as_options( fieldname, @@ -2039,6 +2054,8 @@ frappe.ui.form.Form = class FrappeForm { } show_submission_queue_banner() { + let wrapper = this.layout.wrapper.find(".submission-queue-banner"); + if ( !( this.meta.is_submittable && @@ -2046,10 +2063,15 @@ frappe.ui.form.Form = class FrappeForm { !this.doc.__islocal && this.doc.docstatus === 0 ) - ) - return; + ) { + if (wrapper.length) { + wrapper.hide(); + wrapper.html(""); + } + + return; + } - let wrapper = this.layout.wrapper.find(".submission-queue-banner"); if (!wrapper.length) { wrapper = $('
'); this.layout.wrapper.prepend(wrapper); @@ -2070,7 +2092,7 @@ frappe.ui.form.Form = class FrappeForm { col_width = 3; failed_link = `
${__( - "Latest Failed Submission" + "Previous Failed Submission" )}
`; } @@ -2081,7 +2103,7 @@ frappe.ui.form.Form = class FrappeForm { ${__("Submission Status:")}
- ${__("Latest Submission")} + ${__("Previous Submission")}
${failed_link}
From 84c8cb87059c4ba85f4131560e74b082b7a9d6b3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 31 Oct 2022 19:16:00 +0530 Subject: [PATCH 079/167] test: Added more tests --- frappe/tests/test_query.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index f7fff6fbf3..fb85bcfe25 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -280,5 +280,37 @@ class TestQuery(FrappeTestCase): ancestors_result = list(itertools.chain.from_iterable(ancestors_result)) self.assertListEqual(ancestors_result, get_ancestors_of("Test Tree DocType", "Child 2")) + not_descendants_result = frappe.qb.engine.get_query( + "Test Tree DocType", + fields=["name"], + filters={"name": ("not descendants of", "Parent 1")}, + orderby="modified", + ).run(as_dict=1) + + self.assertListEqual( + not_descendants_result, + frappe.db.get_all( + "Test Tree DocType", + fields=["name"], + filters={"name": ("not descendants of", "Parent 1")}, + ), + ) + + not_ancestors_result = frappe.qb.engine.get_query( + "Test Tree DocType", + fields=["name"], + filters={"name": ("not ancestors of", "Child 2")}, + orderby="modified", + ).run(as_dict=1) + + self.assertListEqual( + not_ancestors_result, + frappe.db.get_all( + "Test Tree DocType", + fields=["name"], + filters={"name": ("not ancestors of", "Child 2")}, + ), + ) + frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") From 8e118773f30e1305c4ea8476f191eeba46754d7f Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 1 Nov 2022 14:33:33 +0530 Subject: [PATCH 080/167] refactor: joining tables --- frappe/database/query.py | 62 +++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 6b3d42cfa8..5ee49639c9 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -650,10 +650,30 @@ class Engine: fields.extend(function_objects) return fields - def join_(self, criterion, fields, table, join): + def join_(self, criterion: Criterion, join_type, table_to_join_on: Table, primary_table: Table): + if self.joined_tables.get(join_type) != table_to_join_on: + criterion = getattr(criterion, join_type)(table_to_join_on).on( + table_to_join_on.parent == primary_table.name + ) + self.joined_tables[join_type] = table_to_join_on + return criterion + + def join(self, criterion, fields, table, join_type): """Handles all join operations on criterion objects""" has_join = False - joined_tables = {} + + def _update_pypika_fields(field): + if not is_pypika_function_object(field): + field = field if isinstance(field, (str, PseudoColumn)) else field.get_sql() + if not TABLE_PATTERN.search(str(field)): + if isinstance(field, PseudoColumn): + field = field.get_sql() + return getattr(frappe.qb.DocType(table), field) + else: + return field + else: + field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args] + return field if not isinstance(fields, Criterion): for field in fields: @@ -665,36 +685,21 @@ class Engine: ): has_join = True table_to_join_on = table_from_string(str(field)) - if joined_tables.get(join) != table_to_join_on: - criterion = getattr(criterion, join)(table_to_join_on).on( - getattr(table_to_join_on, "parent") == getattr(frappe.qb.DocType(table), "name") - ) - joined_tables[join] = table_to_join_on + primary_table = frappe.qb.DocType(table) if not isinstance(table, Table) else table + criterion = self.join_(criterion, join_type, table_to_join_on, primary_table) if has_join: - - def _update_pypika_fields(field): - if not is_pypika_function_object(field): - field = field if isinstance(field, (str, PseudoColumn)) else field.get_sql() - if not TABLE_PATTERN.search(str(field)): - if isinstance(field, PseudoColumn): - field = field.get_sql() - return getattr(frappe.qb.DocType(table), field) - else: - return field - else: - field.args = [getattr(frappe.qb.DocType(table), arg.get_sql()) for arg in field.args] - return field - fields = [_update_pypika_fields(field) for field in fields] if len(self.tables) > 1: primary_table = self.tables.pop(table) for table_object in self.tables.values(): - if joined_tables.get("left_join") != table_object: - criterion = getattr(criterion, join)(table_object).on( - table_object.parent == primary_table.name - ) + criterion = self.join_( + criterion, + join_type=join_type, + table_to_join_on=table_object, + primary_table=primary_table, + ) return criterion, fields @@ -707,13 +712,16 @@ class Engine: ) -> MySQLQueryBuilder | PostgreSQLQueryBuilder: # Clean up state before each query self.tables = {} + self.joined_tables = {} self.linked_doctype = None self.fieldname = None fields = self.set_fields(fields, **kwargs) criterion = self.build_conditions(table, filters, **kwargs) - join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" - criterion, fields = self.join_(criterion=criterion, fields=fields, table=table, join=join) + join_type = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" + criterion, fields = self.join( + criterion=criterion, fields=fields, table=table, join_type=join_type + ) if isinstance(fields, (list, tuple)): query = criterion.select(*fields) From e5d19d78d7427bb17577386df72bcba05701b4d5 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 1 Nov 2022 14:41:46 +0530 Subject: [PATCH 081/167] feat: considering parenttype while joining --- frappe/database/query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 5ee49639c9..a7d8c324c5 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -653,7 +653,8 @@ class Engine: def join_(self, criterion: Criterion, join_type, table_to_join_on: Table, primary_table: Table): if self.joined_tables.get(join_type) != table_to_join_on: criterion = getattr(criterion, join_type)(table_to_join_on).on( - table_to_join_on.parent == primary_table.name + (table_to_join_on.parent == primary_table.name) + & (table_to_join_on.parenttype == primary_table._table_name.replace("tab", "")) ) self.joined_tables[join_type] = table_to_join_on return criterion From 97b0b743ae2edf820839aabfc59e347d372bb65d Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 1 Nov 2022 14:44:53 +0530 Subject: [PATCH 082/167] test: Updated tests for parenttype condition --- frappe/tests/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 6465d4566d..adee42baa5 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -21,7 +21,7 @@ class TestQuery(FrappeTestCase): ["DocType", "parent", "=", "something"], ], ).get_sql(), - "SELECT * FROM `tabDocType` LEFT JOIN `tabBOM Update Log` ON `tabBOM Update Log`.`parent`=`tabDocType`.`name` WHERE `tabBOM Update Log`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'", + "SELECT * FROM `tabDocType` LEFT JOIN `tabBOM Update Log` ON `tabBOM Update Log`.`parent`=`tabDocType`.`name` AND `tabBOM Update Log`.`parenttype`='DocType' WHERE `tabBOM Update Log`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'", ) @run_only_if(db_type_is.MARIADB) From 0bc2ba05458cb79af18dfa47c0c84dda6b3618d5 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 2 Nov 2022 02:39:19 +0530 Subject: [PATCH 083/167] refactor: better naming --- frappe/database/query.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index a7d8c324c5..8da6bde68d 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -650,13 +650,15 @@ class Engine: fields.extend(function_objects) return fields - def join_(self, criterion: Criterion, join_type, table_to_join_on: Table, primary_table: Table): - if self.joined_tables.get(join_type) != table_to_join_on: - criterion = getattr(criterion, join_type)(table_to_join_on).on( - (table_to_join_on.parent == primary_table.name) - & (table_to_join_on.parenttype == primary_table._table_name.replace("tab", "")) + def join_( + self, criterion: Criterion, join_type: str, child_table: Table, parent_table: Table + ) -> Criterion: + if self.joined_tables.get(join_type) != child_table: + criterion = getattr(criterion, join_type)(child_table).on( + (child_table.parent == parent_table.name) + & (child_table.parenttype == parent_table._table_name.replace("tab", "")) ) - self.joined_tables[join_type] = table_to_join_on + self.joined_tables[join_type] = child_table return criterion def join(self, criterion, fields, table, join_type): @@ -685,21 +687,26 @@ class Engine: and (f"`tab{table}`" not in str(field)) ): has_join = True - table_to_join_on = table_from_string(str(field)) - primary_table = frappe.qb.DocType(table) if not isinstance(table, Table) else table - criterion = self.join_(criterion, join_type, table_to_join_on, primary_table) + child_table = table_from_string(str(field)) + parent_table = frappe.qb.DocType(table) if not isinstance(table, Table) else table + criterion = self.join_( + criterion=criterion, + join_type=join_type, + child_table=child_table, + parent_table=parent_table, + ) if has_join: fields = [_update_pypika_fields(field) for field in fields] if len(self.tables) > 1: - primary_table = self.tables.pop(table) - for table_object in self.tables.values(): + parent_table = self.tables.pop(table) + for child_table in self.tables.values(): criterion = self.join_( criterion, join_type=join_type, - table_to_join_on=table_object, - primary_table=primary_table, + child_table=child_table, + parent_table=parent_table, ) return criterion, fields From 58c262a3ed2aa14ebe8be14483e85bd0c7ab22a5 Mon Sep 17 00:00:00 2001 From: Ponnusamy <95607086+Ponnusamy1-V@users.noreply.github.com> Date: Wed, 2 Nov 2022 12:27:50 +0530 Subject: [PATCH 084/167] List view -> add custom buttons to menu --- frappe/public/js/frappe/ui/page.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/public/js/frappe/ui/page.js b/frappe/public/js/frappe/ui/page.js index 0c15eef774..4e67272f88 100644 --- a/frappe/public/js/frappe/ui/page.js +++ b/frappe/public/js/frappe/ui/page.js @@ -600,6 +600,11 @@ frappe.ui.Page = class Page { let response = action(); me.btn_disable_enable(btn, response); }; + // Add actions as menu item in Mobile View + let menu_item_label = group ? `${group} > ${label}` : label; + let menu_item = this.add_menu_item(menu_item_label, _action, false); + menu_item.parent().addClass("hidden-xl"); + if (group) { var $group = this.get_or_add_inner_group_button(group); $(this.inner_toolbar).removeClass("hide"); From 28a124ca47791d4a42d02668d524c2f83d5b3d37 Mon Sep 17 00:00:00 2001 From: Daizy Date: Sat, 22 Oct 2022 13:31:34 +0530 Subject: [PATCH 085/167] perf: cache document naming rule to avoid multiple db call --- frappe/model/naming.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index d9dc0ee48c..163633b54e 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -235,13 +235,21 @@ def set_naming_from_document_naming_rule(doc): if doc.doctype in log_types: return - # ignore_ddl if naming is not yet bootstrapped - for d in frappe.get_all( - "Document Naming Rule", - dict(document_type=doc.doctype, disabled=0), - order_by="priority desc", - ignore_ddl=True, - ): + def _get_document_naming_rule(): + # ignore_ddl if naming is not yet bootstrapped + + return frappe.get_all( + "Document Naming Rule", + {"document_type": doc.doctype, "disabled": 0}, + order_by="priority desc", + ignore_ddl=True, + ) + + document_naming_rules = frappe.cache().hget( + "document_naming_rule", doc.doctype, _get_document_naming_rule + ) + + for d in document_naming_rules: frappe.get_cached_doc("Document Naming Rule", d.name).apply(doc) if doc.name: break From 78d30905ac34a294a2f0db946c85f05987a47eac Mon Sep 17 00:00:00 2001 From: Daizy Date: Mon, 24 Oct 2022 13:32:48 +0530 Subject: [PATCH 086/167] refactor: `get_doctype_map()` using single query and use generator for caching --- frappe/cache_manager.py | 29 +++++++------------ .../document_naming_rule.py | 6 ++++ frappe/model/naming.py | 17 ++++------- 3 files changed, 22 insertions(+), 30 deletions(-) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index d4ce92f384..5862d532d8 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -8,11 +8,14 @@ from frappe.desk.notifications import clear_notifications, delete_notification_c common_default_keys = ["__default", "__global"] -doctype_map_keys = ( - "energy_point_rule_map", - "assignment_rule_map", - "milestone_tracker_map", -) +doctypes_for_mapping = { + "Energy Point Rule", + "Assignment Rule", + "Milestone Tracker", + "Document Naming Rule", +} + +doctype_map_keys = tuple(f"{frappe.scrub(doctype)}_map" for doctype in doctypes_for_mapping) bench_cache_keys = ("assets_json",) @@ -163,21 +166,11 @@ def clear_controller_cache(doctype=None): def get_doctype_map(doctype, name, filters=None, order_by=None): cache = frappe.cache() cache_key = frappe.scrub(doctype) + "_map" - doctype_map = cache.hget(cache_key, name) - if doctype_map is not None: - # cached, return - items = json.loads(doctype_map) - else: - # non cached, build cache - try: - items = frappe.get_all(doctype, filters=filters, order_by=order_by) - cache.hset(cache_key, name, json.dumps(items)) - except frappe.db.TableMissingError: - # executed from inside patch, ignore - items = [] + def _get_items(): + return frappe.get_all(doctype, filters=filters, order_by=order_by, ignore_ddl=True) - return items + return cache.hget(cache_key, name, _get_items) def clear_doctype_map(doctype, name): diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 3fecf26ade..6d32893ceb 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -12,6 +12,12 @@ class DocumentNamingRule(Document): def validate(self): self.validate_fields_in_conditions() + def on_update(self): + frappe.cache_manager.clear_doctype_map("Document Naming Rule", self.document_type) + + def on_trash(self): + frappe.cache_manager.clear_doctype_map("Document Naming Rule", self.document_type) + def validate_fields_in_conditions(self): if self.has_value_changed("document_type"): docfields = [x.fieldname for x in frappe.get_meta(self.document_type).fields] diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 163633b54e..6f7db5cf9d 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -235,18 +235,11 @@ def set_naming_from_document_naming_rule(doc): if doc.doctype in log_types: return - def _get_document_naming_rule(): - # ignore_ddl if naming is not yet bootstrapped - - return frappe.get_all( - "Document Naming Rule", - {"document_type": doc.doctype, "disabled": 0}, - order_by="priority desc", - ignore_ddl=True, - ) - - document_naming_rules = frappe.cache().hget( - "document_naming_rule", doc.doctype, _get_document_naming_rule + document_naming_rules = frappe.cache_manager.get_doctype_map( + "Document Naming Rule", + doc.doctype, + filters={"document_type": doc.doctype, "disabled": 0}, + order_by="priority desc", ) for d in document_naming_rules: From 380a638f3cd8182e857f87497bb73672b4acfaac Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 28 Oct 2022 16:18:39 +0530 Subject: [PATCH 087/167] chore: refactor duplicate code --- frappe/cache_manager.py | 21 +++++++++++-------- .../document_naming_rule.py | 7 +++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 5862d532d8..47b753833a 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -15,7 +15,12 @@ doctypes_for_mapping = { "Document Naming Rule", } -doctype_map_keys = tuple(f"{frappe.scrub(doctype)}_map" for doctype in doctypes_for_mapping) + +def get_doctype_map_key(doctype): + return frappe.scrub(doctype) + "_map" + + +doctype_map_keys = tuple(map(get_doctype_map_key, doctypes_for_mapping)) bench_cache_keys = ("assets_json",) @@ -40,7 +45,7 @@ global_cache_keys = ( "sitemap_routes", "db_tables", "server_script_autocompletion_items", -) + doctype_map_keys +) user_cache_keys = ( "bootinfo", @@ -164,13 +169,11 @@ def clear_controller_cache(doctype=None): def get_doctype_map(doctype, name, filters=None, order_by=None): - cache = frappe.cache() - cache_key = frappe.scrub(doctype) + "_map" - - def _get_items(): - return frappe.get_all(doctype, filters=filters, order_by=order_by, ignore_ddl=True) - - return cache.hget(cache_key, name, _get_items) + return frappe.cache().hget( + get_doctype_map_key(doctype), + name, + lambda: frappe.get_all(doctype, filters=filters, order_by=order_by, ignore_ddl=True), + ) def clear_doctype_map(doctype, name): diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 6d32893ceb..3da2d0105a 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -12,11 +12,14 @@ class DocumentNamingRule(Document): def validate(self): self.validate_fields_in_conditions() - def on_update(self): + def clear_doctype_map(self): frappe.cache_manager.clear_doctype_map("Document Naming Rule", self.document_type) + def on_update(self): + self.clear_doctype_map() + def on_trash(self): - frappe.cache_manager.clear_doctype_map("Document Naming Rule", self.document_type) + self.clear_doctype_map() def validate_fields_in_conditions(self): if self.has_value_changed("document_type"): From 85f6f1e01df375692aade92ac55e89b8e9a989b4 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 28 Oct 2022 16:25:46 +0530 Subject: [PATCH 088/167] fix: dont clear doctype map keys when clearing doctype cache --- frappe/cache_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 47b753833a..eeddef1865 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -45,7 +45,7 @@ global_cache_keys = ( "sitemap_routes", "db_tables", "server_script_autocompletion_items", -) +) + doctype_map_keys user_cache_keys = ( "bootinfo", @@ -74,7 +74,7 @@ doctype_cache_keys = ( "notifications", "workflow", "data_import_column_header_map", -) + doctype_map_keys +) def clear_user_cache(user=None): From e82e4d1a7344808bbff9a566fda76a6d71f4496a Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 28 Oct 2022 16:35:20 +0530 Subject: [PATCH 089/167] fix: use property instead of hardcoding --- .../core/doctype/document_naming_rule/document_naming_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 3da2d0105a..598de98dbb 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -13,7 +13,7 @@ class DocumentNamingRule(Document): self.validate_fields_in_conditions() def clear_doctype_map(self): - frappe.cache_manager.clear_doctype_map("Document Naming Rule", self.document_type) + frappe.cache_manager.clear_doctype_map(self.doctype, self.document_type) def on_update(self): self.clear_doctype_map() From 88d506864b536cb498424f6849e63247979e533d Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 2 Nov 2022 17:21:37 +0530 Subject: [PATCH 090/167] refactor: consistent quries and better naming --- frappe/database/query.py | 27 +++++++----- frappe/tests/test_query.py | 87 +++++++++++++++++++------------------- 2 files changed, 61 insertions(+), 53 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 8da6bde68d..5fe99b42a2 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -172,10 +172,13 @@ def table_from_string(table: str) -> "DocType": def get_nested_set_hierarchy_result(hierarchy: str, field: str, table: str): ref_doctype = table - lft, rgt = "", "" - lft, rgt = ( - frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == field).run()[0] - ) + try: + lft, rgt = ( + frappe.qb.from_(ref_doctype).select("lft", "rgt").where(Field("name") == field).run()[0] + ) + except IndexError: + lft, rgt = None, None + if hierarchy in ("descendants of", "not descendants of"): result = ( frappe.qb.from_(ref_doctype) @@ -591,7 +594,7 @@ class Engine: if " as " in field: field, reference = field.split(" as ") if "`" in field: - updated_fields.append(PseudoColumn(f"{field} as {reference}")) + updated_fields.append(PseudoColumn(f"{field} {reference}")) else: updated_fields.append(Field(field.strip()).as_(reference)) elif "`" in str(field): @@ -650,13 +653,17 @@ class Engine: fields.extend(function_objects) return fields - def join_( - self, criterion: Criterion, join_type: str, child_table: Table, parent_table: Table + def join_child_tables( + self, + criterion: Criterion, + join_type: str, + child_table: Table, + parent_table: Table, ) -> Criterion: if self.joined_tables.get(join_type) != child_table: criterion = getattr(criterion, join_type)(child_table).on( (child_table.parent == parent_table.name) - & (child_table.parenttype == parent_table._table_name.replace("tab", "")) + & (child_table.parenttype == TAB_PATTERN.sub("", parent_table._table_name)) ) self.joined_tables[join_type] = child_table return criterion @@ -689,7 +696,7 @@ class Engine: has_join = True child_table = table_from_string(str(field)) parent_table = frappe.qb.DocType(table) if not isinstance(table, Table) else table - criterion = self.join_( + criterion = self.join_child_tables( criterion=criterion, join_type=join_type, child_table=child_table, @@ -702,7 +709,7 @@ class Engine: if len(self.tables) > 1: parent_table = self.tables.pop(table) for child_table in self.tables.values(): - criterion = self.join_( + criterion = self.join_child_tables( criterion, join_type=join_type, child_table=child_table, diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index adee42baa5..320e5e9cc7 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -9,6 +9,49 @@ from frappe.tests.utils import FrappeTestCase from frappe.utils.nestedset import get_ancestors_of, get_descendants_of +def create_tree_docs(): + records = [ + { + "some_fieldname": "Root Node", + "parent_test_tree_doctype": None, + "is_group": 1, + }, + { + "some_fieldname": "Parent 1", + "parent_test_tree_doctype": "Root Node", + "is_group": 1, + }, + { + "some_fieldname": "Parent 2", + "parent_test_tree_doctype": "Root Node", + "is_group": 1, + }, + { + "some_fieldname": "Child 1", + "parent_test_tree_doctype": "Parent 1", + "is_group": 0, + }, + { + "some_fieldname": "Child 2", + "parent_test_tree_doctype": "Parent 1", + "is_group": 0, + }, + { + "some_fieldname": "Child 3", + "parent_test_tree_doctype": "Parent 2", + "is_group": 0, + }, + ] + + tree_doctype = new_doctype("Test Tree DocType", is_tree=True, autoname="field:some_fieldname") + tree_doctype.insert() + + for record in records: + d = frappe.new_doc("Test Tree DocType") + d.update(record) + d.insert() + + class TestQuery(FrappeTestCase): @run_only_if(db_type_is.MARIADB) def test_multiple_tables_in_filters(self): @@ -225,52 +268,10 @@ class TestQuery(FrappeTestCase): "email", frappe.qb.engine.get_query("User", fields=["name", "#email"], filters={}).get_sql() ) - def insert_tree_docs(self): - records = [ - { - "some_fieldname": "Root Node", - "parent_test_tree_doctype": None, - "is_group": 1, - }, - { - "some_fieldname": "Parent 1", - "parent_test_tree_doctype": "Root Node", - "is_group": 1, - }, - { - "some_fieldname": "Parent 2", - "parent_test_tree_doctype": "Root Node", - "is_group": 1, - }, - { - "some_fieldname": "Child 1", - "parent_test_tree_doctype": "Parent 1", - "is_group": 0, - }, - { - "some_fieldname": "Child 2", - "parent_test_tree_doctype": "Parent 1", - "is_group": 0, - }, - { - "some_fieldname": "Child 3", - "parent_test_tree_doctype": "Parent 2", - "is_group": 0, - }, - ] - - tree_doctype = new_doctype("Test Tree DocType", is_tree=True, autoname="field:some_fieldname") - tree_doctype.insert() - - for record in records: - d = frappe.new_doc("Test Tree DocType") - d.update(record) - d.insert() - def test_nestedset(self): frappe.db.sql("delete from `tabDocType` where `name` = 'Test Tree DocType'") frappe.db.sql_ddl("drop table if exists `tabTest Tree DocType`") - self.insert_tree_docs() + create_tree_docs() descendants_result = frappe.qb.engine.get_query( "Test Tree DocType", fields=["name"], From e1719fd30c6af3022c131cf2cc5b98b39232eeaa Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 3 Nov 2022 03:56:59 +0530 Subject: [PATCH 091/167] feat: Added support for table_field.fieldname and tests --- frappe/database/query.py | 33 +++++++++++++++------------------ frappe/tests/test_query.py | 21 +++++++++++---------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 5fe99b42a2..1d0447e512 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -553,18 +553,14 @@ class Engine: alias = None if " as " in field: field, alias = field.split(" as ") - self.fieldname, linked_fieldname = field.split(".") - linked_field = frappe.get_meta(doctype, cached=True).get_field(self.fieldname) - try: - self.linked_doctype = linked_field.options - except AttributeError: - return fields - field = f"`tab{self.linked_doctype}`.`{linked_fieldname}`" - if alias: - field = f"{field} as {alias}" - _fields.append(field) + fieldname, linked_fieldname = field.split(".") + linked_doctype = frappe.get_meta(doctype).get_field(fieldname).options - return _fields + field = f"`tab{linked_doctype}`.`{linked_fieldname}`" + if alias: + field = f"{field} {alias}" + _fields.append(field) + return _fields def sanitize_fields(self, fields: str | list | tuple): is_mariadb = frappe.db.db_type == "mariadb" @@ -584,11 +580,11 @@ class Engine: return fields - def get_list_fields(self, fields: list) -> list: + def get_list_fields(self, table: str, fields: list) -> list: updated_fields = [] if issubclass(type(fields), Criterion) or "*" in fields: return fields - # fields = self.get_fieldnames_from_child_table(doctype=table, fields=fields) + fields = self.get_fieldnames_from_child_table(doctype=table, fields=fields) for field in fields: if not isinstance(field, Criterion) and field: if " as " in field: @@ -616,7 +612,7 @@ class Engine: fields = Field(fields).as_(reference) return fields - def set_fields(self, fields, **kwargs) -> list: + def set_fields(self, table: str, fields, **kwargs) -> list: fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name" fields = self.sanitize_fields(fields) if isinstance(fields, list) and None in fields and Field not in fields: @@ -644,7 +640,7 @@ class Engine: if is_str: fields = self.get_string_fields(fields) if not is_str and fields: - fields = self.get_list_fields(fields) + fields = self.get_list_fields(table, fields) # Need to check instance again since fields modified. if not isinstance(fields, (list, tuple, set)): @@ -707,8 +703,9 @@ class Engine: fields = [_update_pypika_fields(field) for field in fields] if len(self.tables) > 1: - parent_table = self.tables.pop(table) - for child_table in self.tables.values(): + parent_table = self.tables[table] + child_tables = list(self.tables.values())[1:] + for child_table in child_tables: criterion = self.join_child_tables( criterion, join_type=join_type, @@ -731,8 +728,8 @@ class Engine: self.linked_doctype = None self.fieldname = None - fields = self.set_fields(fields, **kwargs) criterion = self.build_conditions(table, filters, **kwargs) + fields = self.set_fields(table, fields, **kwargs) join_type = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" criterion, fields = self.join( criterion=criterion, fields=fields, table=table, join_type=join_type diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 320e5e9cc7..a0a9c921da 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -241,12 +241,8 @@ class TestQuery(FrappeTestCase): "Note", filters={"name": "Test Note Title"}, fields=["name", "`tabNote Seen By`.`user` as seen_by"], - ).run(as_dict=1), - frappe.get_list( - "Note", - filters={"name": "Test Note Title"}, - fields=["name", "`tabNote Seen By`.`user` as seen_by"], - ), + ).get_sql(), + "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'", ) self.assertEqual( @@ -254,12 +250,17 @@ class TestQuery(FrappeTestCase): "Note", filters={"name": "Test Note Title"}, fields=["name", "`tabNote Seen By`.`user` as seen_by", "`tabNote Seen By`.`idx` as idx"], - ).run(as_dict=1), - frappe.get_list( + ).get_sql(), + "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'", + ) + + self.assertEqual( + frappe.qb.engine.get_query( "Note", filters={"name": "Test Note Title"}, - fields=["name", "`tabNote Seen By`.`user` as seen_by", "`tabNote Seen By`.`idx` as idx"], - ), + fields=["name", "seen_by.user as seen_by", "`tabNote Seen By`.`idx` as idx"], + ).get_sql(), + "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'", ) @run_only_if(db_type_is.MARIADB) From dfe62f261470f3f27f581850252e4cbb19cb87fa Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 3 Nov 2022 12:49:00 +0530 Subject: [PATCH 092/167] test: use double quotes for postgres query assertions --- frappe/tests/test_query.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index a0a9c921da..3f48882345 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -242,7 +242,9 @@ class TestQuery(FrappeTestCase): filters={"name": "Test Note Title"}, fields=["name", "`tabNote Seen By`.`user` as seen_by"], ).get_sql(), - "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'", + "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'".replace( + "`", '"' if frappe.db.db_type == "postgres" else "`" + ), ) self.assertEqual( @@ -251,7 +253,9 @@ class TestQuery(FrappeTestCase): filters={"name": "Test Note Title"}, fields=["name", "`tabNote Seen By`.`user` as seen_by", "`tabNote Seen By`.`idx` as idx"], ).get_sql(), - "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'", + "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'".replace( + "`", '"' if frappe.db.db_type == "postgres" else "`" + ), ) self.assertEqual( @@ -260,7 +264,9 @@ class TestQuery(FrappeTestCase): filters={"name": "Test Note Title"}, fields=["name", "seen_by.user as seen_by", "`tabNote Seen By`.`idx` as idx"], ).get_sql(), - "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'", + "SELECT `tabNote`.`name`,`tabNote Seen By`.`user` seen_by,`tabNote Seen By`.`idx` idx FROM `tabNote` LEFT JOIN `tabNote Seen By` ON `tabNote Seen By`.`parent`=`tabNote`.`name` AND `tabNote Seen By`.`parenttype`='Note' WHERE `tabNote`.`name`='Test Note Title'".replace( + "`", '"' if frappe.db.db_type == "postgres" else "`" + ), ) @run_only_if(db_type_is.MARIADB) From e725ae333eaa0f7432b45abe6abdd915c828fcd1 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 1 Nov 2022 15:16:00 +0530 Subject: [PATCH 093/167] fix: dont repeatedly use get_doc for unlocking reference doc * chore: add weird behaviour note on unlock_doc method --- .../core/doctype/submission_queue/submission_queue.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index c0748ebf58..7156de8eee 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -130,8 +130,10 @@ class SubmissionQueue(Document): # assuming the job failed here (?) status = "failed" + queued_doc = self.queued_doc + # Job finished successfully however action was never completed (?) - if status == "finished" and self.queued_doc.docstatus != 1: + if status == "finished" and queued_doc.docstatus == 0: status = "failed" # Checking if job is queue to be executed/executing @@ -140,12 +142,16 @@ class SubmissionQueue(Document): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): - self.queued_doc.unlock() + queued_doc.unlock() frappe.db.set_value("Submission Queue", self.name, "status", "Failed", update_modified=False) frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() def unlock_doc(self): + # NOTE: this can lead to some weird unlocking/locking behaviours. + # for example: hitting unlock on a submission could lead to unlocking of another submission + # of the same reference document. + if self.status != "Queued": return From 8b73108270e6d5ea5e68891c34ffb33fa6bc44ff Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 3 Nov 2022 20:43:24 +0530 Subject: [PATCH 094/167] feat: added PseudoColumnMapper for postgres support --- frappe/database/query.py | 38 ++++++++++++++++++++--------------- frappe/query_builder/utils.py | 10 +++++++++ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 1d0447e512..a9dab02744 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -15,8 +15,7 @@ from frappe.database.utils import NestedSetHierarchy, is_pypika_function_object from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table, functions from frappe.query_builder.functions import Function, SqlFunctions -from frappe.query_builder.utils import PseudoColumn -from frappe.utils import cstr +from frappe.query_builder.utils import PseudoColumnMapper from frappe.utils.data import MARIADB_SPECIFIC_COMMENT if TYPE_CHECKING: @@ -27,7 +26,6 @@ WORDS_PATTERN = re.compile(r"\w+") BRACKETS_PATTERN = re.compile(r"\(.*?\)|$") SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions] COMMA_PATTERN = re.compile(r",\s*(?![^()]*\))") -TABLE_PATTERN = re.compile(r"`\btab\w+") def like(key: Field, value: str) -> frappe.qb: @@ -166,8 +164,11 @@ def has_function(field): def table_from_string(table: str) -> "DocType": - table_name = table.split("`", maxsplit=1)[1].split(".")[0][3:] - return frappe.qb.DocType(table_name=table_name.replace("`", "")) + if frappe.db.db_type == "postgres": + table_name = table.split('"', maxsplit=1)[1].split(".")[0][3:].replace('"', "") + else: + table_name = table.split("`", maxsplit=1)[1].split(".")[0][3:].replace("`", "") + return frappe.qb.DocType(table_name=table_name) def get_nested_set_hierarchy_result(hierarchy: str, field: str, table: str): @@ -467,13 +468,15 @@ class Engine: has_primitive_operator = True field = operator_mapping( *map( - lambda field: Field(field.strip()) if "`" not in field else PseudoColumn(field.strip()), + lambda field: Field(field.strip()) + if "`" not in field + else PseudoColumnMapper(field.strip()), arg.split(_operator), ), ) field = ( - (Field(initial_fields) if "`" not in initial_fields else PseudoColumn(initial_fields)) + (Field(initial_fields) if "`" not in initial_fields else PseudoColumnMapper(initial_fields)) if not has_primitive_operator else field ) @@ -590,11 +593,11 @@ class Engine: if " as " in field: field, reference = field.split(" as ") if "`" in field: - updated_fields.append(PseudoColumn(f"{field} {reference}")) + updated_fields.append(PseudoColumnMapper(f"{field} {reference}")) else: updated_fields.append(Field(field.strip()).as_(reference)) elif "`" in str(field): - updated_fields.append(PseudoColumn(field.strip())) + updated_fields.append(PseudoColumnMapper(field.strip())) else: updated_fields.append(Field(field)) return updated_fields @@ -603,11 +606,11 @@ class Engine: if fields == "*": return fields if "`" in fields: - fields = PseudoColumn(fields) + fields = PseudoColumnMapper(fields) if " as " in str(fields): fields, reference = str(fields).split(" as ") if "`" in str(fields): - fields = PseudoColumn(f"{fields} as {reference}") + fields = PseudoColumnMapper(f"{fields} {reference}") else: fields = Field(fields).as_(reference) return fields @@ -667,12 +670,15 @@ class Engine: def join(self, criterion, fields, table, join_type): """Handles all join operations on criterion objects""" has_join = False + table_pattern = ( + re.compile(r"`\btab\w+") if frappe.db.db_type == "mariadb" else re.compile(r'"\btab\w+') + ) def _update_pypika_fields(field): if not is_pypika_function_object(field): - field = field if isinstance(field, (str, PseudoColumn)) else field.get_sql() - if not TABLE_PATTERN.search(str(field)): - if isinstance(field, PseudoColumn): + field = field if isinstance(field, (str, PseudoColumnMapper)) else field.get_sql() + if not table_pattern.search(str(field)): + if isinstance(field, PseudoColumnMapper): field = field.get_sql() return getattr(frappe.qb.DocType(table), field) else: @@ -686,8 +692,8 @@ class Engine: # Only perform this bit if foreign doctype in fields if ( not is_pypika_function_object(field) - and str(field).startswith("`tab") - and (f"`tab{table}`" not in str(field)) + and (str(field).startswith('"tab') or str(field).startswith("`tab")) + and (f"`tab{table}`" not in str(field) and f'tab{table}"' not in str(field)) ): has_join = True child_table = table_from_string(str(field)) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index f0130ca813..8780841e03 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -12,6 +12,16 @@ from frappe.query_builder.terms import NamedParameterWrapper from .builder import MariaDB, Postgres +class PseudoColumnMapper(PseudoColumn): + def __init__(self, name: str) -> None: + super().__init__(name) + + def get_sql(self, **kwargs): + if frappe.db.db_type == "postgres": + self.name = self.name.replace("`", '"') + return self.name + + class db_type_is(Enum): MARIADB = "mariadb" POSTGRES = "postgres" From 0ed7600604e9574cc0e5841e38f3dd8333645e8d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 3 Nov 2022 20:49:05 +0530 Subject: [PATCH 095/167] build: py3.11 support (#18550) * chore: bump RestrictedPython * ci: use py3.11 on development version * fix: py311 compat changes * chore: bump to final versions --- .github/workflows/linters.yml | 4 ++-- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/server-mariadb-tests.yml | 2 +- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- frappe/utils/background_jobs.py | 4 ++-- pyproject.toml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 6c3ba7db81..1d8e736538 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -42,7 +42,7 @@ jobs: - name: 'Setup Environment' uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v3 - name: Validate Docs @@ -79,7 +79,7 @@ jobs: steps: - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v3 - run: | pip install pip-audit diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 12bf9eca55..4feaebe15d 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -19,7 +19,7 @@ jobs: node-version: 16 - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up bench and build assets run: | npm install -g yarn diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 9c2b933763..ae101d003b 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -63,7 +63,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Check for valid Python & Merge Conflicts run: | diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 926a87249f..dcc078ad2a 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -66,7 +66,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Check for valid Python & Merge Conflicts run: | diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index f41171784c..8a78d2e750 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -64,7 +64,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Check for valid Python & Merge Conflicts run: | diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index e88cd75efb..d416857588 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -3,7 +3,7 @@ import socket import time from collections import defaultdict from functools import lru_cache -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Union from uuid import uuid4 import redis @@ -61,7 +61,7 @@ def enqueue( *, at_front=False, **kwargs, -) -> "Job" | Any: +) -> Union["Job", Any]: """ Enqueue method to be executed using a background worker diff --git a/pyproject.toml b/pyproject.toml index dd51deed3e..6f744ca186 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "PyPika~=0.48.9", "PyQRCode~=1.2.1", "PyYAML~=5.4.1", - "RestrictedPython~=5.2", + "RestrictedPython~=6.0", "WeasyPrint==52.5", "Werkzeug~=2.2.2", "Whoosh~=2.7.4", From 8b84042037fede43e029d97c30afecf271d25b89 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 4 Nov 2022 05:14:17 +0000 Subject: [PATCH 096/167] fix: load `doc_before_save` in `check_if_latest` (#18752) --- frappe/model/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 2a2f924b00..8dcc57e827 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -245,7 +245,6 @@ class Document(BaseDocument): self._set_defaults() self.set_user_and_timestamp() self.set_docstatus() - self.load_doc_before_save() self.check_if_latest() self._validate_links() self.check_permission("create") @@ -326,7 +325,6 @@ class Document(BaseDocument): self.set_user_and_timestamp() self.set_docstatus() - self.load_doc_before_save() self.check_if_latest() self.set_parent_in_children() self.set_name_in_children() @@ -746,6 +744,8 @@ class Document(BaseDocument): Will also validate document transitions (Save > Submit > Cancel) calling `self.check_docstatus_transition`.""" + self.load_doc_before_save() + self._action = "save" previous = self.get_doc_before_save() From 9aa55263053ea07a5ecd9e2127ed63eadced9b53 Mon Sep 17 00:00:00 2001 From: Maharshi Patel Date: Fri, 4 Nov 2022 12:07:39 +0530 Subject: [PATCH 097/167] fix: child table don't run mobile specific code on_input_focus & handle_date_picker function has mobile specific code that relies on touch inputs so there is no need to run it on desktop. --- frappe/public/js/frappe/form/grid_row.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 4a30ad68e0..f9f09187bc 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -944,19 +944,21 @@ export default class GridRow { vertical = false; horizontal = false; }) - .on("click", function () { + .on("click", function (event) { if (frappe.ui.form.editable_row !== me) { var out = me.toggle_editable_row(); } var col = this; let first_input_field = $(col).find('input[type="Text"]:first'); - - first_input_field.length && on_input_focus(first_input_field); - first_input_field.trigger("focus"); - first_input_field.one("blur", () => (input_in_focus = false)); - first_input_field.data("fieldtype") == "Date" && handle_date_picker(); + if (event.pointerType == "touch") { + first_input_field.length && on_input_focus(first_input_field); + + first_input_field.one("blur", () => (input_in_focus = false)); + + first_input_field.data("fieldtype") == "Date" && handle_date_picker(); + } return out; }); From f0197d35d6cff2076353753f2ed6ad7bb35ff399 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 4 Nov 2022 15:40:58 +0530 Subject: [PATCH 098/167] refactor: qb builder class identification --- frappe/query_builder/builder.py | 4 ++++ frappe/query_builder/utils.py | 3 +-- frappe/tests/test_query_builder.py | 8 -------- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index 60d0a1208d..5a7b06221f 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -47,6 +47,8 @@ class Base: class MariaDB(Base, MySQLQuery): Field = terms.Field + _BuilderClasss = MySQLQueryBuilder + @classmethod def _builder(cls, *args, **kwargs) -> "MySQLQueryBuilder": return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) @@ -70,6 +72,8 @@ class Postgres(Base, PostgreSQLQuery): # they are two different objects. The quick fix used here is to replace the # Field names in the "Field" function. + _BuilderClasss = PostgreSQLQueryBuilder + @classmethod def _builder(cls, *args, **kwargs) -> "PostgreSQLQueryBuilder": return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index f0130ca813..d3a4935382 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -102,8 +102,7 @@ def patch_query_execute(): raise frappe.PermissionError("Only SELECT SQL allowed in scripting") return query, param_collector.get_parameters() - query_class = get_attr(str(frappe.qb).split("'")[1]) - builder_class = get_type_hints(query_class._builder).get("return") + builder_class = frappe.qb._BuilderClasss if not builder_class: raise BuilderIdentificationFailed diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index b80e3ad8e2..d05c8d7d02 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -373,11 +373,3 @@ class TestMisc(FrappeTestCase): DocType = Table("DocType") self.assertEqual(DocType.get_sql(), "DocType") - - def test_error_on_query_class(self): - import frappe.query_builder.utils - - frappe.query_builder.utils.get_type_hints = lambda x: {"return": None} - - with self.assertRaises(frappe.query_builder.utils.BuilderIdentificationFailed): - frappe.query_builder.utils.patch_query_execute() From f20fa69282c770e3fc91ea5cea973ff412b509a3 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 4 Nov 2022 11:54:26 +0000 Subject: [PATCH 099/167] fix!: remove relaxations for Cordova (#18728) --- frappe/app.py | 6 ++---- frappe/auth.py | 39 +++++++++++++++++---------------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/frappe/app.py b/frappe/app.py index 136b16bff5..8cb32ff4bf 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -12,13 +12,13 @@ from werkzeug.wrappers import Request, Response import frappe import frappe.api -import frappe.auth import frappe.handler import frappe.monitor import frappe.rate_limiter import frappe.recorder import frappe.utils.response from frappe import _ +from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request from frappe.middlewares import StaticDataMiddleware from frappe.utils import get_site_name, sanitize_html @@ -29,8 +29,6 @@ local_manager = LocalManager(frappe.local) _site = None _sites_path = os.environ.get("SITES_PATH", ".") -SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS") -UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH") @local_manager.middleware @@ -118,7 +116,7 @@ def init_request(request): make_form_dict(request) if request.method != "OPTIONS": - frappe.local.http_request = frappe.auth.HTTPRequest() + frappe.local.http_request = HTTPRequest() def setup_read_only_mode(): diff --git a/frappe/auth.py b/frappe/auth.py index f7ff6f0fe5..e4bde99907 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -20,6 +20,9 @@ from frappe.utils import cint, date_diff, datetime, get_datetime, today from frappe.utils.password import check_password from frappe.website.utils import get_home_page +SAFE_HTTP_METHODS = frozenset(("GET", "HEAD", "OPTIONS")) +UNSAFE_HTTP_METHODS = frozenset(("POST", "PUT", "DELETE", "PATCH")) + class HTTPRequest: def __init__(self): @@ -67,25 +70,21 @@ class HTTPRequest: frappe.local.login_manager = LoginManager() def validate_csrf_token(self): - if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"): - if not frappe.local.session: - return - if ( - not frappe.local.session.data.csrf_token - or frappe.local.session.data.device == "mobile" - or frappe.conf.get("ignore_csrf", None) - ): - # not via boot - return + if ( + not frappe.request + or frappe.request.method not in UNSAFE_HTTP_METHODS + or frappe.conf.ignore_csrf + or not frappe.session + or not (saved_token := frappe.session.data.csrf_token) + or ( + (frappe.get_request_header("X-Frappe-CSRF-Token") or frappe.form_dict.pop("csrf_token", None)) + == saved_token + ) + ): + return - csrf_token = frappe.get_request_header("X-Frappe-CSRF-Token") - if not csrf_token and "csrf_token" in frappe.local.form_dict: - csrf_token = frappe.local.form_dict.csrf_token - del frappe.local.form_dict["csrf_token"] - - if frappe.local.session.data.csrf_token != csrf_token: - frappe.local.flags.disable_traceback = True - frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) + frappe.flags.disable_traceback = True + frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): frappe.local.lang = get_language() @@ -354,10 +353,6 @@ class CookieManager: if not secure and hasattr(frappe.local, "request"): secure = frappe.local.request.scheme == "https" - # Cordova does not work with Lax - if frappe.local.session.data.device == "mobile": - samesite = None - self.cookies[key] = { "value": value, "expires": expires, From 032df946bead39b603e1eda6acd2049475bde757 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 5 Nov 2022 14:08:35 +0530 Subject: [PATCH 100/167] test: bg jobs test cleanup (#18767) * test: fix flaky RQ job tests Sometimes stop_job doesn't succeed and causes tests to timeout. Reduced sleep time to avoid this in tests. We are still testing all the important features - monitoring. * build(deps): Bump RQ to latest version Minor bugfixes that affect us ref: https://github.com/rq/rq/releases * test: sanity tests for scheduled job types * test(test_runner): dont set bench_id globally * refactor: stop_job shouldn't throw error The intention of use here is to stop stuck jobs or long running jobs, if for some reason they were stopped by the time command gets executed, there's no need to throw error. --- frappe/core/doctype/rq_job/rq_job.py | 7 +++++- frappe/core/doctype/rq_job/test_rq_job.py | 15 ++++++------- frappe/test_runner.py | 1 - frappe/tests/test_redis.py | 4 ++++ frappe/tests/test_zbg_job_sanity_test.py | 27 +++++++++++++++++++++++ pyproject.toml | 2 +- 6 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 frappe/tests/test_zbg_job_sanity_test.py diff --git a/frappe/core/doctype/rq_job/rq_job.py b/frappe/core/doctype/rq_job/rq_job.py index 7e1c35a0e6..f05611fe7d 100644 --- a/frappe/core/doctype/rq_job/rq_job.py +++ b/frappe/core/doctype/rq_job/rq_job.py @@ -5,10 +5,12 @@ import functools import re from rq.command import send_stop_job_command +from rq.exceptions import InvalidJobOperation from rq.job import Job from rq.queue import Queue import frappe +from frappe import _ from frappe.model.document import Document from frappe.utils import ( cint, @@ -93,7 +95,10 @@ class RQJob(Document): @check_permissions def stop_job(self): - send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id) + try: + send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id) + except InvalidJobOperation: + frappe.msgprint(_("Job is not running."), title=_("Invalid Operation")) @staticmethod def get_count(args) -> int: diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index ae0691fa61..460aa08941 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -19,12 +19,11 @@ class TestRQJob(FrappeTestCase): @timeout(seconds=20) def check_status(self, job: Job, status, wait=True): - if wait: - while True: - if job.is_queued or job.is_started: - time.sleep(0.2) - else: - break + while wait: + if not (job.is_queued or job.is_started): + break + time.sleep(0.2) + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) def test_serialization(self): @@ -69,7 +68,7 @@ class TestRQJob(FrappeTestCase): self.assertGreaterEqual(len(non_failed_jobs), 1) # Create a slow job and check if it's stuck in "Started" - job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=1000) + job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=10) time.sleep(3) self.check_status(job, "started", wait=False) stop_job(job_id=job.id) @@ -84,8 +83,8 @@ class TestRQJob(FrappeTestCase): def test_is_enqueued(self): + dummy_job = frappe.enqueue(self.BG_JOB, sleep=10, queue="short") job_name = "uniq_test_job" - dummy_job = frappe.enqueue(self.BG_JOB, sleep=100, queue="short") actual_job = frappe.enqueue(self.BG_JOB, job_name=job_name, queue="short") self.assertTrue(is_job_queued(job_name)) diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 1e3573336a..7e2c7e724f 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -86,7 +86,6 @@ def main( frappe.utils.scheduler.disable_scheduler() set_test_email_config() - frappe.conf.update({"bench_id": "test_bench", "use_rq_auth": False}) if not frappe.flags.skip_before_tests: if verbose: diff --git a/frappe/tests/test_redis.py b/frappe/tests/test_redis.py index ca59af98b0..19001028f6 100644 --- a/frappe/tests/test_redis.py +++ b/frappe/tests/test_redis.py @@ -1,4 +1,5 @@ import functools +from unittest.mock import patch import redis @@ -30,12 +31,14 @@ def skip_if_redis_version_lt(version): class TestRedisAuth(FrappeTestCase): @skip_if_redis_version_lt("6.0") + @patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False}) def test_rq_gen_acllist(self): """Make sure that ACL list is genrated""" acl_list = RedisQueue.gen_acl_list() self.assertEqual(acl_list[1]["bench"][0], get_bench_id()) @skip_if_redis_version_lt("6.0") + @patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False}) def test_adding_redis_user(self): acl_list = RedisQueue.gen_acl_list() username, password = acl_list[1]["bench"] @@ -47,6 +50,7 @@ class TestRedisAuth(FrappeTestCase): conn.acl_deluser(username) @skip_if_redis_version_lt("6.0") + @patch.dict(frappe.conf, {"bench_id": "test_bench", "use_rq_auth": False}) def test_rq_namespace(self): """Make sure that user can access only their respective namespace.""" # Current bench ID diff --git a/frappe/tests/test_zbg_job_sanity_test.py b/frappe/tests/test_zbg_job_sanity_test.py new file mode 100644 index 0000000000..19dc168c04 --- /dev/null +++ b/frappe/tests/test_zbg_job_sanity_test.py @@ -0,0 +1,27 @@ +""" smoak tests to check that all registered background jobs execute without error. + +Note: Filename is intentional to run this test roughly at end. Don't change.""" + +import time + +import frappe +from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs +from frappe.tests.utils import FrappeTestCase, timeout + + +class TestScheduledJobSanity(FrappeTestCase): + def setUp(self): + remove_failed_jobs() + + @timeout(90) + def test_bg_jobs_run(self): + """Enqueue all scheduled jobs, wait for finish and verify that none failed.""" + for scheduled_job_type in frappe.get_all("Scheduled Job Type", pluck="name"): + frappe.get_doc("Scheduled Job Type", scheduled_job_type).enqueue(force=True) + + while RQJob.get_list({"filters": [["RQ Job", "status", "in", ("queued", "started")]]}): + time.sleep(0.5) + + # Check no failed, if failed print full details + failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]}) + self.assertEqual(len(failed_jobs), 0, "Jobs failed: " + str(failed_jobs)) diff --git a/pyproject.toml b/pyproject.toml index 6f744ca186..348353003c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ dependencies = [ "hiredis~=2.0.0", "requests-oauthlib~=1.3.0", "requests~=2.27.1", - "rq~=1.10.1", + "rq~=1.11.1", "rsa>=4.1", "semantic-version~=2.10.0", "sqlparse~=0.4.1", From 43ccb40d0e4598bf71fe8af2f1fec6dda1e8adc8 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 5 Nov 2022 18:33:50 +0530 Subject: [PATCH 101/167] chore: rollback to 3.10 for vuln checks cython dependency fails [skip ci] --- .github/workflows/linters.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 1d8e736538..01b5407489 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -79,7 +79,7 @@ jobs: steps: - uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.10' - uses: actions/checkout@v3 - run: | pip install pip-audit From 313605d328d192078b41fd196b41af21754373cf Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 5 Nov 2022 20:37:43 +0530 Subject: [PATCH 102/167] fix: standard dashboards not loading --- frappe/public/js/frappe/views/breadcrumbs.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index 5236c21ac2..53a3300a94 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -137,13 +137,13 @@ frappe.breadcrumbs = { const doctype_meta = frappe.get_doc("DocType", doctype); if ( (doctype === "User" && !frappe.user.has_role("System Manager")) || - (doctype_meta && doctype_meta.issingle) + doctype_meta?.issingle ) { // no user listview for non-system managers and single doctypes } else { let route; const doctype_route = frappe.router.slug(frappe.router.doctype_layout || doctype); - if (doctype_meta.is_tree) { + if (doctype_meta?.is_tree) { let view = frappe.model.user_settings[doctype].last_view || "Tree"; route = `${doctype_route}/view/${view}`; } else { From cd1bbccdc3a412478dc9ec1a4cddca937f0109db Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sat, 5 Nov 2022 16:05:37 +0000 Subject: [PATCH 103/167] fix: dont release `frappe.local` twice (#18772) --- frappe/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/app.py b/frappe/app.py index 8cb32ff4bf..889c2e7cbe 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -86,7 +86,7 @@ def application(request: Request): log_request(request, response) process_response(response) - frappe.destroy() + frappe.db.close() return response From ff0c820c9c7c1b9c3173b0c2ce8d1be8fe576b76 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sat, 5 Nov 2022 16:06:07 +0000 Subject: [PATCH 104/167] chore(socketio): remove duplicate event handler (#18770) --- socketio.js | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/socketio.js b/socketio.js index 711e0028cc..30e931d6bf 100644 --- a/socketio.js +++ b/socketio.js @@ -31,11 +31,6 @@ io.on("connection", function (socket) { socket.user = cookie.parse(socket.request.headers.cookie).user_id; - socket.on("task_subscribe", function (task_id) { - var room = get_task_room(socket, task_id); - socket.join(room); - }); - let retries = 0; let join_user_room = () => { request @@ -61,13 +56,13 @@ io.on("connection", function (socket) { join_user_room(); - socket.on("task_unsubscribe", function (task_id) { + socket.on("task_subscribe", function (task_id) { var room = get_task_room(socket, task_id); - socket.leave(room); + socket.join(room); }); socket.on("task_unsubscribe", function (task_id) { - var room = "task:" + task_id; + var room = get_task_room(socket, task_id); socket.leave(room); }); From 5d1d3564ef41ceec652c7f097ae9ef4150420ffa Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 5 Nov 2022 20:45:02 +0530 Subject: [PATCH 105/167] test: Dashboard basic ui test + whitelisting --- cypress/integration/dashboard.js | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 cypress/integration/dashboard.js diff --git a/cypress/integration/dashboard.js b/cypress/integration/dashboard.js new file mode 100644 index 0000000000..6eb28567bc --- /dev/null +++ b/cypress/integration/dashboard.js @@ -0,0 +1,50 @@ +describe("Dashboard view", { scrollBehavior: false }, () => { + before(() => { + cy.login(); + cy.visit("/app"); + }); + + it("should load", () => { + const chart = "TODO-YEARLY-TRENDS"; + const dashboard = "TODO-TEST-DASHBOARD"; // check slash in name intentionally. + + cy.insert_doc( + "Dashboard Chart", + { + is_standard: 0, + chart_name: chart, + chart_type: "Count", + document_type: "ToDo", + parent_document_type: "", + based_on: "creation", + group_by_type: "Count", + timespan: "Last Year", + time_interval: "Yearly", + timeseries: 1, + type: "Line", + filters_json: "[]", + }, + true + ); + + cy.insert_doc( + "Dashboard", + { + name: dashboard, + dashboard_name: dashboard, + is_standard: 0, + charts: [ + { + chart: chart, + }, + ], + }, + true + ); + + cy.visit(`/app/dashboard-view/${dashboard}`); + + // expect chart to be loaded + cy.findByText(chart).should("be.visible"); + }); +}); From 9a7e59e811366f5ea0aefc7d4a9d1e4e387920b0 Mon Sep 17 00:00:00 2001 From: Ponnusamy <95607086+Ponnusamy1-V@users.noreply.github.com> Date: Sun, 6 Nov 2022 15:00:53 +0530 Subject: [PATCH 106/167] fix: dashboard view from workspace --- frappe/public/js/frappe/views/breadcrumbs.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index 53a3300a94..2eaa70fe0f 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -29,7 +29,7 @@ frappe.breadcrumbs = { return localStorage["preferred_breadcrumbs:" + doctype]; }, - add(module, doctype, type) { + async add(module, doctype, type) { let obj; if (typeof module === "object") { obj = module; @@ -40,7 +40,7 @@ frappe.breadcrumbs = { type: type, }; } - + await frappe.model.with_doctype(doctype); this.all[frappe.breadcrumbs.current_page()] = obj; this.update(); }, From 09d35c74eb77f76cdca073bda3833160f1d83c79 Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Sun, 6 Nov 2022 12:57:02 +0100 Subject: [PATCH 107/167] fix: remove redundant translation (#18775) --- frappe/client.py | 5 ----- frappe/core/page/permission_manager/permission_manager.py | 2 -- frappe/desk/desk_page.py | 5 ----- frappe/desk/query_report.py | 5 ----- 4 files changed, 17 deletions(-) diff --git a/frappe/client.py b/frappe/client.py index f42f73a529..404617b68c 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -333,11 +333,6 @@ def get_js(items): with open(contentpath) as srcfile: code = frappe.utils.cstr(srcfile.read()) - if frappe.local.lang != "en": - messages = frappe.get_lang_dict("jsfile", contentpath) - messages = json.dumps(messages) - code += f"\n\n$.extend(frappe._messages, {messages})" - out.append(code) return out diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 46c9e0aca2..45c1e44fa1 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -19,7 +19,6 @@ from frappe.permissions import ( setup_custom_perms, update_permission_property, ) -from frappe.translate import send_translations from frappe.utils.user import get_users_with_role as _get_user_with_role not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"] @@ -28,7 +27,6 @@ not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Tran @frappe.whitelist() def get_roles_and_doctypes(): frappe.only_for("System Manager") - send_translations(frappe.get_lang_dict("doctype", "DocPerm")) active_domains = frappe.get_active_domains() diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index ad0bd549d8..bde27125f6 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -2,7 +2,6 @@ # License: MIT. See LICENSE import frappe -from frappe.translate import send_translations @frappe.whitelist() @@ -31,10 +30,6 @@ def getpage(): page = frappe.form_dict.get("name") doc = get(page) - # load translations - if frappe.lang != "en": - send_translations(frappe.get_lang_dict("page", page)) - frappe.response.docs.append(doc) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 877fdbe5bc..d0bc63f858 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -14,7 +14,6 @@ from frappe.model.utils import render_include from frappe.modules import get_module_path, scrub from frappe.monitor import add_data_to_monitor from frappe.permissions import get_role_permissions -from frappe.translate import send_translations from frappe.utils import ( cint, cstr, @@ -202,10 +201,6 @@ def get_script(report_name): if not script: script = "frappe.query_reports['%s']={}" % report_name - # load translations - if frappe.lang != "en": - send_translations(frappe.get_lang_dict("report", report_name)) - return { "script": render_include(script), "html_format": html_format, From e02b90cd5b0be4d6dd40742fdeff2ac01e70b638 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 6 Nov 2022 17:33:02 +0530 Subject: [PATCH 108/167] fix: dont allow reading attributes of unsafe objects (#18706) --- frappe/tests/test_safe_exec.py | 16 ++++++++++++++ frappe/utils/safe_exec.py | 39 ++++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/frappe/tests/test_safe_exec.py b/frappe/tests/test_safe_exec.py index caa98737a2..fcd5832680 100644 --- a/frappe/tests/test_safe_exec.py +++ b/frappe/tests/test_safe_exec.py @@ -1,3 +1,5 @@ +import types + import frappe from frappe.tests.utils import FrappeTestCase from frappe.utils.safe_exec import get_safe_globals, safe_exec @@ -59,3 +61,17 @@ class TestSafeExec(FrappeTestCase): # enqueue whitelisted method safe_exec("""frappe.enqueue("ping", now=True)""") + + def test_ensure_getattrable_globals(self): + def check_safe(objects): + for obj in objects: + if isinstance(obj, (types.ModuleType, types.CodeType, types.TracebackType, types.FrameType)): + self.fail(f"{obj} wont work in safe exec.") + elif isinstance(obj, dict): + check_safe(obj.values()) + + check_safe(get_safe_globals().values()) + + def test_unsafe_objects(self): + unsafe_global = {"frappe": frappe} + self.assertRaises(SyntaxError, safe_exec, """frappe.msgprint("Hello")""", unsafe_global) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index afdd4694a8..c75a5fd12b 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -2,6 +2,9 @@ import copy import inspect import json import mimetypes +import types +from contextlib import contextmanager +from functools import lru_cache import RestrictedPython.Guards from RestrictedPython import compile_restricted, safe_globals @@ -64,14 +67,20 @@ def safe_exec(script, _globals=None, _locals=None, restrict_commit_rollback=Fals exec_globals.frappe.db.pop("rollback", None) exec_globals.frappe.db.pop("add_index", None) - # execute script compiled by RestrictedPython - frappe.flags.in_safe_exec = True - exec(compile_restricted(script), exec_globals, _locals) # pylint: disable=exec-used - frappe.flags.in_safe_exec = False + with safe_exec_flags(), patched_qb(): + # execute script compiled by RestrictedPython + exec(compile_restricted(script), exec_globals, _locals) # pylint: disable=exec-used return exec_globals, _locals +@contextmanager +def safe_exec_flags(): + frappe.flags.in_safe_exec = True + yield + frappe.flags.in_safe_exec = False + + def get_safe_globals(): datautils = frappe._dict() @@ -258,6 +267,24 @@ def call_with_form_dict(function, kwargs): frappe.local.form_dict = form_dict +@contextmanager +def patched_qb(): + try: + _terms = frappe.qb.terms + frappe.qb.terms = _flatten(frappe.qb.terms) + yield + finally: + frappe.qb.terms = _terms + + +@lru_cache +def _flatten(module): + new_mod = NamespaceDict() + for name, obj in inspect.getmembers(module, lambda x: not inspect.ismodule(x)): + new_mod[name] = obj + return new_mod + + def get_python_builtins(): return { "abs": abs, @@ -350,6 +377,10 @@ def _getattr(object, name, default=None): if isinstance(name, str) and (name in UNSAFE_ATTRIBUTES): raise SyntaxError(f"{name} is an unsafe attribute") + + if isinstance(object, (types.ModuleType, types.CodeType, types.TracebackType, types.FrameType)): + raise SyntaxError(f"Reading {object} attributes is not allowed") + return RestrictedPython.Guards.safer_getattr(object, name, default=default) From e3a0b007c21858d19e86d2aa26e55737eefe6d44 Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Sun, 6 Nov 2022 13:06:51 +0100 Subject: [PATCH 109/167] fix: translations (#18765) * fix: add missing translations * fix: capitalize label * fix: translate label on summary item --- frappe/public/js/frappe/utils/utils.js | 2 +- frappe/public/js/frappe/views/reports/query_report.js | 2 +- frappe/translations/de.csv | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 3d17e8a8f6..cf5b619c37 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1383,7 +1383,7 @@ Object.assign(frappe.utils, { : ""; return $(`
- ${summary.label} + ${__(summary.label)}
${value}
`); }, diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 1febf11b6c..dc5e285181 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1067,7 +1067,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { { fieldname: "sb_1", fieldtype: "Section Break", - label: "Y axis", + label: "Y Axis", }, { fieldname: "y_axis_fields", diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index 78fdcb7925..b1000f3bee 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -4080,7 +4080,7 @@ Scheduler Event,Scheduler-Ereignis, Select Event Type,Wählen Sie den Ereignistyp, Schedule Script,Zeitplan-Skript, Duration,Dauer, -Donut,Krapfen, +Donut,Donut, Custom Options,Benutzerdefinierte Optionen, "Ex: ""colors"": [""#d1d8dd"", ""#ff5858""]","Beispiel: "Farben": ["# d1d8dd", "# ff5858"]", Confirmation Email Template,Bestätigungs-E-Mail-Vorlage, @@ -4818,3 +4818,8 @@ K,Tsd,Number system M,Mio,Number system B,Mrd,Number system T,Bio,Number system +Type of Chart,Diagrammtyp, +Preview Chart,Vorschau erzeugen, +Please select X and Y fields,Bitte Felder für die X- und Y-Achse wählen, +Notification sent to,Benachrichtigung gesendet an, +Add to this activity by mailing to {0},"Senden Sie eine E-Mail an {0}, damit sie hier erscheint", From db1ed206f310603e05e30f0c0c33f5a83e831dba Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 6 Nov 2022 18:53:09 +0530 Subject: [PATCH 110/167] fix: only release db if it exists ref https://github.com/frappe/frappe/pull/18772 [skip ci] --- frappe/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/app.py b/frappe/app.py index 889c2e7cbe..0d7fdc1fe1 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -86,7 +86,8 @@ def application(request: Request): log_request(request, response) process_response(response) - frappe.db.close() + if frappe.db: + frappe.db.close() return response From 44a5bdc3f1d6b184797c41946aad29f9286ca289 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 6 Nov 2022 19:18:30 +0530 Subject: [PATCH 111/167] fix: ignore internal methods (#18784) --- frappe/utils/safe_exec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index c75a5fd12b..04aa134d39 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -281,7 +281,8 @@ def patched_qb(): def _flatten(module): new_mod = NamespaceDict() for name, obj in inspect.getmembers(module, lambda x: not inspect.ismodule(x)): - new_mod[name] = obj + if not name.startswith("_"): + new_mod[name] = obj return new_mod From 62c4a3c020dcd58550c6e36e1ab32730b4ff4b99 Mon Sep 17 00:00:00 2001 From: Daizy Modi Date: Sun, 6 Nov 2022 19:59:59 +0530 Subject: [PATCH 112/167] fix: check permission before print or download document (#18757) --- frappe/utils/weasyprint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/utils/weasyprint.py b/frappe/utils/weasyprint.py index ceb064a955..d670cf96f5 100644 --- a/frappe/utils/weasyprint.py +++ b/frappe/utils/weasyprint.py @@ -9,6 +9,7 @@ import frappe @frappe.whitelist() def download_pdf(doctype, name, print_format, letterhead=None): doc = frappe.get_doc(doctype, name) + doc.check_permission("print") generator = PrintFormatGenerator(print_format, doc, letterhead) pdf = generator.render_pdf() @@ -21,6 +22,7 @@ def download_pdf(doctype, name, print_format, letterhead=None): def get_html(doctype, name, print_format, letterhead=None): doc = frappe.get_doc(doctype, name) + doc.check_permission("print") generator = PrintFormatGenerator(print_format, doc, letterhead) return generator.get_html_preview() From f04287ae83ed1531212c50defe52b75264d44d80 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 6 Nov 2022 20:21:04 +0530 Subject: [PATCH 113/167] fix: dont close static modal with keyboard closes https://github.com/frappe/frappe/issues/11926 [skip ci] --- frappe/public/js/frappe/ui/keyboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/keyboard.js b/frappe/public/js/frappe/ui/keyboard.js index 14418528a6..2e3bd95616 100644 --- a/frappe/public/js/frappe/ui/keyboard.js +++ b/frappe/public/js/frappe/ui/keyboard.js @@ -331,7 +331,7 @@ function close_grid_and_dialog() { } // close open dialog - if (cur_dialog && !cur_dialog.no_cancel_flag) { + if (cur_dialog && !cur_dialog.no_cancel_flag && !cur_dialog.static) { cur_dialog.cancel(); return false; } From 074d880dceebdb2fdb83553030961f6b074700e1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 7 Nov 2022 03:19:15 +0530 Subject: [PATCH 114/167] test: Testing for completion --- .../submission_queue/test_submission_queue.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index 3ec99f06b2..51ac15db86 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -1,21 +1,47 @@ # Copyright (c) 2022, Frappe Technologies and Contributors # See license.txt +import time +import typing + import frappe -from frappe.tests.utils import FrappeTestCase +from frappe.tests.utils import FrappeTestCase, timeout from frappe.utils.background_jobs import get_queue +if typing.TYPE_CHECKING: + from rq.job import Job + class TestSubmissionQueue(FrappeTestCase): queue = get_queue(qtype="default") + @timeout(seconds=20) + def check_status(self, job: "Job", status, wait=True): + if wait: + while True: + if job.is_queued or job.is_started: + time.sleep(0.2) + else: + break + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) + def test_queue_operation(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.core.doctype.submission_queue.submission_queue import queue_submission - doc = frappe.get_doc({"doctype": "ToDo", "description": "Something"}).insert() - queue_submission(doc, "submit") + doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) + doc.insert() + + d = frappe.new_doc("Test Submission Queue") + d.update({"some_fieldname": "Random"}) + d.insert() + + queue_submission(d, "submit") submission_queue = frappe.get_last_doc("Submission Queue") # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) self.assertIn(job.get_status(refresh=True), ("queued", "started")) + + # Test completion + self.check_status(job, status="finished") From d84c4b28eb23adbde0b93ffd8d29a59e13fb5cd8 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 7 Nov 2022 17:10:25 +0530 Subject: [PATCH 115/167] feat: Adding exception to submisison queue while unlocking doc --- .../doctype/submission_queue/submission_queue.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 7156de8eee..4aa51df746 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -5,6 +5,7 @@ from datetime import datetime from rq.exceptions import NoSuchJobError from rq.job import Job +from rq.registry import FailedJobRegistry import frappe from frappe import _ @@ -12,7 +13,7 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_queue, get_redis_conn from frappe.utils.data import cint @@ -68,7 +69,6 @@ class SubmissionQueue(Document): self, to_be_queued_doc: Document, action_for_queuing: str, enqueued_by_session_id: str ): _action = action_for_queuing.lower() - if _action == "update": _action = "submit" @@ -143,7 +143,15 @@ class SubmissionQueue(Document): # Checking any one of the possible termination statuses elif status in ("failed", "canceled", "stopped"): queued_doc.unlock() - frappe.db.set_value("Submission Queue", self.name, "status", "Failed", update_modified=False) + values = {"status": "Failed"} + + # Defining job exception when unlocking document. + registry = FailedJobRegistry(queue=get_queue(qtype="default")) + for job_id in registry.get_job_ids(): + if job_id == self.job_id: + values = {"status": "Failed", "exception": job.exc_info} + + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() From b2860e6f9ed45214631f655880899227d075f1e9 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 7 Nov 2022 19:01:50 +0530 Subject: [PATCH 116/167] fix(UX): allow clicking on row to open in new tab (#18789) - ctrl+click on row is toggling checkbox instead of opening it in new row - This only happens on non-mac devices Root cause: incorrect grouping of predicate towards fixing https://github.com/frappe/frappe/issues/18788 [skip ci] --- frappe/public/js/frappe/list/list_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 66ff5107a1..52d0026e37 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1179,7 +1179,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { this.$result.on("click", ".list-row, .image-view-header, .file-header", (e) => { const $target = $(e.target); // tick checkbox if Ctrl/Meta key is pressed - if (e.ctrlKey || (e.metaKey && !$target.is("a"))) { + if ((e.ctrlKey || e.metaKey) && !$target.is("a")) { const $list_row = $(e.currentTarget); const $check = $list_row.find(".list-row-checkbox"); $check.prop("checked", !$check.prop("checked")); From 4ec69e9384813a2c0b6b4272d55fdc43892078e9 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 8 Nov 2022 12:55:33 +0530 Subject: [PATCH 117/167] refactor: using after commit in enqueuing & removed session time logic --- .../submission_queue/submission_queue.py | 55 +++++++------------ 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 4aa51df746..1153a9a8a2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,9 +1,7 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt -from datetime import datetime - -from rq.exceptions import NoSuchJobError +from rq import get_current_job from rq.job import Job from rq.registry import FailedJobRegistry @@ -12,7 +10,7 @@ from frappe import _ from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification from frappe.model.document import Document from frappe.monitor import add_data_to_monitor -from frappe.utils import DATETIME_FORMAT, now, time_diff_in_seconds +from frappe.utils import now, time_diff_in_seconds from frappe.utils.background_jobs import get_queue, get_redis_conn from frappe.utils.data import cint @@ -38,10 +36,9 @@ class SubmissionQueue(Document): table = frappe.qb.DocType("Submission Queue") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) - def insert(self, to_be_queued_doc: Document, action: str, session_id: str): + def insert(self, to_be_queued_doc: Document, action: str): self.to_be_queued_doc = to_be_queued_doc self.action_for_queuing = action - self.enqueued_by_session_id = session_id super().insert(ignore_permissions=True) def lock(self): @@ -51,23 +48,24 @@ class SubmissionQueue(Document): self.queued_doc.unlock() def after_insert(self): - job = self.queue_action( + self.queue_action( "background_submission", to_be_queued_doc=self.queued_doc, action_for_queuing=self.action_for_queuing, - enqueued_by_session_id=self.enqueued_by_session_id, timeout=600, + enqueue_after_commit=True, ) + + def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): + current_job = get_current_job() + + # Set the job id for that submission doctype frappe.db.set_value( self.doctype, self.name, - {"job_id": job.id}, + {"job_id": current_job.id}, update_modified=False, ) - - def background_submission( - self, to_be_queued_doc: Document, action_for_queuing: str, enqueued_by_session_id: str - ): _action = action_for_queuing.lower() if _action == "update": _action = "submit" @@ -82,9 +80,9 @@ class SubmissionQueue(Document): values["ended_at"] = now() frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - self.notify(values["status"], action_for_queuing, enqueued_by_session_id) + self.notify(values["status"], action_for_queuing) - def notify(self, submission_status: str, action: str, session_id: str): + def notify(self, submission_status: str, action: str): if submission_status == "Failed": doctype = self.doctype docname = self.name @@ -98,7 +96,7 @@ class SubmissionQueue(Document): frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - if cint(time_diff_in_seconds(now(), get_user_last_request_time(session_id))) <= 60: + if cint(time_diff_in_seconds(self.created_at, now())) <= 60: frappe.publish_realtime( "msgprint", { @@ -120,13 +118,9 @@ class SubmissionQueue(Document): notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") enqueue_create_notification([notify_to], notification_doc) - frappe.publish_realtime(f"reload_doc_{self.ref_doctype}_{self.ref_docname}") - def _unlock_reference_doc(self): - try: - job = Job.fetch(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) - except NoSuchJobError: + job_id = frappe.db.get_value(self.doctype, self.name, "job_id") + if not job_id: # assuming the job failed here (?) status = "failed" @@ -147,8 +141,10 @@ class SubmissionQueue(Document): # Defining job exception when unlocking document. registry = FailedJobRegistry(queue=get_queue(qtype="default")) - for job_id in registry.get_job_ids(): - if job_id == self.job_id: + # If job id is None then this job won't exist in the failed registry + for jid in registry.get_job_ids(): + if jid == job_id: + job = Job.fetch(job_id, connection=get_redis_conn()) values = {"status": "Failed", "exception": job.exc_info} frappe.db.set_value(self.doctype, self.name, values, update_modified=False) @@ -166,21 +162,12 @@ class SubmissionQueue(Document): self._unlock_reference_doc() -def get_user_last_request_time(session_id): - return ( - frappe.cache() - .hget("session", session_id) - .get("data", {}) - .get("last_updated", datetime.min.strftime(DATETIME_FORMAT)) - ) - - def queue_submission(doc: Document, action: str): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.ref_doctype = doc.doctype queue.ref_docname = doc.name - queue.insert(doc, action, frappe.session.sid) + queue.insert(doc, action) frappe.msgprint( _("Queued for Submission. You can track the progress over {0}.").format( From ce360b6fcea2b10b0b2fa297746fc23595bb631f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 5 Nov 2022 16:58:01 +0530 Subject: [PATCH 118/167] feat: Set default SQL statement timeouts --- frappe/database/database.py | 36 ++++++++++++++++++++++++++-- frappe/database/mariadb/database.py | 7 ++++++ frappe/database/postgres/database.py | 8 +++++++ frappe/tests/test_db.py | 21 ++++++++++++++++ frappe/utils/background_jobs.py | 1 - 5 files changed, 70 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 3cb47e853a..64ef994a50 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -7,7 +7,7 @@ import random import re import string import traceback -from contextlib import contextmanager +from contextlib import contextmanager, suppress from time import time from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder @@ -29,7 +29,7 @@ from frappe.exceptions import DoesNotExistError, ImplicitCommitError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.utils import cast as cast_fieldtype -from frappe.utils import get_datetime, get_table_name, getdate, now, sbool +from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") @@ -114,6 +114,17 @@ class Database: self._cursor = self._conn.cursor() frappe.local.rollback_observers = [] + try: + if execution_timeout := get_ideal_query_execution_timeout(): + self.set_execution_timeout(execution_timeout) + except Exception as e: + frappe.logger("database").warning(f"Couldn't set execution timeout {e}") + + def set_execution_timeout(self, seconds: int): + """Set session speicifc timeout on exeuction of statements. + If any statement takes more time it will be killed along with entire transaction.""" + raise NotImplementedError + def use(self, db_name): """`USE` db_name.""" self._conn.select_db(db_name) @@ -1340,3 +1351,24 @@ def savepoint(catch: type | tuple[type, ...] = Exception): frappe.db.rollback(save_point=savepoint) else: frappe.db.release_savepoint(savepoint) + + +def get_ideal_query_execution_timeout() -> int: + """Get execution timeout based on current timeout in contexts. + + HTTP requests: HTTP timeout or a default (300) + Background jobs: Job timeout + + Note: Timeout adds 1.5x as "safety factor" + """ + from rq import get_current_job + + # Zero means no timeout, which is the default value in db. + timeout = 0 + with suppress(Exception): + if hasattr(frappe.local, "request"): + timeout = frappe.conf.http_timeout or 300 + elif job := get_current_job(): + timeout = job.timeout + + return int(cint(timeout) * 1.5) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 1df9877eb1..322c355357 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -68,6 +68,10 @@ class MariaDBExceptionUtil: def is_syntax_error(e: pymysql.Error) -> bool: return e.args[0] == ER.PARSE_ERROR + @staticmethod + def is_statement_timeout(e: pymysql.Error) -> bool: + return e.args[0] == 1969 + @staticmethod def is_data_too_long(e: pymysql.Error) -> bool: return e.args[0] == ER.DATA_TOO_LONG @@ -102,6 +106,9 @@ class MariaDBConnectionUtil: def create_connection(self): return pymysql.connect(**self.get_connection_settings()) + def set_execution_timeout(self, seconds: int): + self.sql("set session max_statement_time = %s", int(seconds)) + def get_connection_settings(self) -> dict: conn_settings = { "host": self.host, diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 3b3612c0e4..d082afceaf 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -99,6 +99,10 @@ class PostgresExceptionUtil: def is_duplicate_fieldname(e): return getattr(e, "pgcode", None) == DUPLICATE_COLUMN + @staticmethod + def is_statement_timeout(e): + return PostgresDatabase.is_timedout(e) or isinstance(e, frappe.QueryTimeoutError) + @staticmethod def is_data_too_long(e): return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION @@ -161,6 +165,10 @@ class PostgresDatabase(PostgresExceptionUtil, Database): return conn + def set_execution_timeout(self, seconds: int): + # Postgres expects milliseconds as input + self.sql("set local statement_timeout = %s", int(seconds) * 1000) + def escape(self, s, percent=True): """Escape quotes and percent in given string.""" if isinstance(s, bytes): diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 08fef66bd0..9a7d086252 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -36,6 +36,27 @@ class TestDB(FrappeTestCase): def test_get_database_size(self): self.assertIsInstance(frappe.db.get_database_size(), (float, int)) + def test_db_statement_execution_timeout(self): + frappe.db.set_execution_timeout(2) + # Setting 0 means no timeout. + self.addCleanup(frappe.db.set_execution_timeout, 0) + + try: + savepoint = "statement_timeout" + frappe.db.savepoint(savepoint) + frappe.db.multisql( + { + "mariadb": "select sleep(10)", + "postgres": "select pg_sleep(10)", + } + ) + except Exception as e: + self.assertTrue(frappe.db.is_statement_timeout(e), f"exepcted {e} to be timeout error") + frappe.db.rollback(save_point=savepoint) + else: + frappe.db.rollback(save_point=savepoint) + self.fail("Long running queries not timing out") + def test_get_value(self): self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator") diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index d416857588..12c2105df8 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -9,7 +9,6 @@ from uuid import uuid4 import redis from redis.exceptions import BusyLoadingError, ConnectionError from rq import Connection, Queue, Worker -from rq.command import send_stop_job_command from rq.logutils import setup_loghandlers from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed From f45656217e13f0e062707d98c3c01ef2479c50a0 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 8 Nov 2022 14:58:25 +0530 Subject: [PATCH 119/167] refactor: control statement timeout via flag --- frappe/database/database.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 64ef994a50..1934bc8c30 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -115,7 +115,7 @@ class Database: frappe.local.rollback_observers = [] try: - if execution_timeout := get_ideal_query_execution_timeout(): + if execution_timeout := get_query_execution_timeout(): self.set_execution_timeout(execution_timeout) except Exception as e: frappe.logger("database").warning(f"Couldn't set execution timeout {e}") @@ -1353,20 +1353,24 @@ def savepoint(catch: type | tuple[type, ...] = Exception): frappe.db.release_savepoint(savepoint) -def get_ideal_query_execution_timeout() -> int: - """Get execution timeout based on current timeout in contexts. +def get_query_execution_timeout() -> int: + """Get execution timeout based on current timeout in different contexts. - HTTP requests: HTTP timeout or a default (300) - Background jobs: Job timeout + HTTP requests: HTTP timeout or a default (300) + Background jobs: Job timeout + Console/Commands: No timeout = 0. - Note: Timeout adds 1.5x as "safety factor" + Note: Timeout adds 1.5x as "safety factor" """ from rq import get_current_job + if not frappe.conf.get("enable_db_statement_timeout"): + return 0 + # Zero means no timeout, which is the default value in db. timeout = 0 with suppress(Exception): - if hasattr(frappe.local, "request"): + if getattr(frappe.local, "request", None): timeout = frappe.conf.http_timeout or 300 elif job := get_current_job(): timeout = job.timeout From 1c44a65ce8d6193b8b5e61f83f320a303cd9da72 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 8 Nov 2022 20:30:37 +0530 Subject: [PATCH 120/167] test: fixing tests --- .../submission_queue/test_submission_queue.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py index 51ac15db86..c057bd22e1 100644 --- a/frappe/core/doctype/submission_queue/test_submission_queue.py +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -29,19 +29,23 @@ class TestSubmissionQueue(FrappeTestCase): from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.core.doctype.submission_queue.submission_queue import queue_submission - doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) - doc.insert() + if not frappe.db.table_exists("Test Submission Queue", cached=False): + doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) + doc.insert() d = frappe.new_doc("Test Submission Queue") d.update({"some_fieldname": "Random"}) d.insert() + frappe.db.commit() queue_submission(d, "submit") + frappe.db.commit() + + # Waiting for execution + time.sleep(4) submission_queue = frappe.get_last_doc("Submission Queue") # Test queueing / starting job = self.queue.fetch_job(submission_queue.job_id) - self.assertIn(job.get_status(refresh=True), ("queued", "started")) - # Test completion self.check_status(job, status="finished") From ad7c0816f8e5f455aeb869132dda46980fe78966 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 8 Nov 2022 21:27:41 +0530 Subject: [PATCH 121/167] fix: prevent deleting standard doctypes in prod (#18803) --- frappe/core/doctype/doctype/test_doctype.py | 3 +++ frappe/installer.py | 4 ++-- frappe/model/delete_doc.py | 2 ++ frappe/tests/test_modules.py | 2 +- frappe/tests/test_virtual_doctype.py | 2 +- frappe/tests/ui_test_helpers.py | 4 ++-- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 3722e5d1fa..2e74fd3a6a 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -670,6 +670,9 @@ class TestDocType(FrappeTestCase): self.assertEqual(test_json.test_json_field["hello"], "world") + def test_no_delete_doc(self): + self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address") + @patch.dict(frappe.conf, {"developer_mode": 1}) def test_custom_field_deletion(self): """Custom child tables whose doctype doesn't exist should be auto deleted.""" diff --git a/frappe/installer.py b/frappe/installer.py index 4f1755c2a0..2a6c29a17f 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -402,7 +402,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]: if not dry_run: if doctype.issingle: - frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True) + frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True, force=True) else: drop_doctypes.append(doctype.name) @@ -460,7 +460,7 @@ def _delete_doctypes(doctypes: list[str], dry_run: bool) -> None: for doctype in set(doctypes): print(f"* dropping Table for '{doctype}'...") if not dry_run: - frappe.delete_doc("DocType", doctype, ignore_on_trash=True) + frappe.delete_doc("DocType", doctype, ignore_on_trash=True, force=True) frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{doctype}`") diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index d1120cc22d..5e8a12c345 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -92,6 +92,8 @@ def delete_doc( else: doc = frappe.get_doc(doctype, name) + if not (doc.custom or frappe.conf.developer_mode or frappe.flags.in_patch or force): + frappe.throw(_("Standard DocType can not be deleted.")) update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) diff --git a/frappe/tests/test_modules.py b/frappe/tests/test_modules.py index d6145afcf4..43c90456b8 100644 --- a/frappe/tests/test_modules.py +++ b/frappe/tests/test_modules.py @@ -69,7 +69,7 @@ class TestUtils(FrappeTestCase): delattr(self, "note") if self._testMethodName == "test_make_boilerplate": - self.doctype.delete() + self.doctype.delete(force=True) scrubbed = frappe.scrub(self.doctype.name) self.addCleanup( delete_path, diff --git a/frappe/tests/test_virtual_doctype.py b/frappe/tests/test_virtual_doctype.py index 22910cb64f..898c5e118c 100644 --- a/frappe/tests/test_virtual_doctype.py +++ b/frappe/tests/test_virtual_doctype.py @@ -87,7 +87,7 @@ class TestVirtualDoctypes(FrappeTestCase): cls.addClassCleanup(frappe.flags.pop, "allow_doctype_export", None) vdt = new_doctype(name=TEST_DOCTYPE_NAME, is_virtual=1, custom=0).insert() - cls.addClassCleanup(vdt.delete) + cls.addClassCleanup(vdt.delete, force=True) patch_virtual_doc = patch( "frappe.controllers", new={frappe.local.site: {TEST_DOCTYPE_NAME: VirtualDoctypeTest}} diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 297d9f5b12..ecb6b4da97 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -451,7 +451,7 @@ def create_test_user(): @frappe.whitelist() def setup_tree_doctype(): - frappe.delete_doc_if_exists("DocType", "Custom Tree") + frappe.delete_doc_if_exists("DocType", "Custom Tree", force=True) frappe.get_doc( { @@ -475,7 +475,7 @@ def setup_tree_doctype(): @frappe.whitelist() def setup_image_doctype(): - frappe.delete_doc_if_exists("DocType", "Custom Image") + frappe.delete_doc_if_exists("DocType", "Custom Image", force=True) frappe.get_doc( { From bbad2b19ebdd76e31010674116aff37861df5ac5 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 8 Nov 2022 21:32:28 +0530 Subject: [PATCH 122/167] feat: Adding more data to monitor --- frappe/core/doctype/submission_queue/submission_queue.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 1153a9a8a2..ef81cff2c6 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -72,7 +72,12 @@ class SubmissionQueue(Document): try: getattr(to_be_queued_doc, _action)() - add_data_to_monitor(doctype=to_be_queued_doc.doctype, action=_action) + add_data_to_monitor( + doctype=to_be_queued_doc.doctype, + action=_action, + execution_time=cint(time_diff_in_seconds(now(), self.created_at)), + enqueued_by=self.enqueued_by, + ) values = {"status": "Finished"} except Exception: values = {"status": "Failed", "exception": frappe.get_traceback()} @@ -96,7 +101,7 @@ class SubmissionQueue(Document): frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - if cint(time_diff_in_seconds(self.created_at, now())) <= 60: + if cint(time_diff_in_seconds(now(), self.created_at)) <= 60: frappe.publish_realtime( "msgprint", { From ad18f82007831097992be6df33d31ba430aff162 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 9 Nov 2022 01:10:17 +0530 Subject: [PATCH 123/167] feat: Added execution time to monitor --- .../doctype/submission_queue/submission_queue.json | 2 +- .../core/doctype/submission_queue/submission_queue.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 00e841b926..7d2a1e0703 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -86,7 +86,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-10-30 19:22:20.998753", + "modified": "2022-11-09 01:01:27.185383", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index ef81cff2c6..063b80f9e9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -74,8 +74,9 @@ class SubmissionQueue(Document): getattr(to_be_queued_doc, _action)() add_data_to_monitor( doctype=to_be_queued_doc.doctype, + docname=to_be_queued_doc.name, action=_action, - execution_time=cint(time_diff_in_seconds(now(), self.created_at)), + execution_time=time_diff_in_seconds(now(), self.created_at), enqueued_by=self.enqueued_by, ) values = {"status": "Finished"} @@ -100,13 +101,14 @@ class SubmissionQueue(Document): message = message.format( frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action) ) - - if cint(time_diff_in_seconds(now(), self.created_at)) <= 60: + time_diff = time_diff_in_seconds(now(), self.created_at) + if cint(time_diff) <= 60: frappe.publish_realtime( "msgprint", { "message": message - + f". View it here", + + f". View it here" + + f". Execution time {round(float(time_diff), 2)} seconds", "alert": True, "indicator": "red" if submission_status == "Failed" else "green", }, @@ -128,7 +130,6 @@ class SubmissionQueue(Document): if not job_id: # assuming the job failed here (?) status = "failed" - queued_doc = self.queued_doc # Job finished successfully however action was never completed (?) From 15cae83c50b202e53c78c8b7de8ac56698c1e117 Mon Sep 17 00:00:00 2001 From: Ponnusamy <95607086+Ponnusamy1-V@users.noreply.github.com> Date: Wed, 9 Nov 2022 13:13:39 +0530 Subject: [PATCH 124/167] Update breadcrumbs.js --- frappe/public/js/frappe/views/breadcrumbs.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index 2eaa70fe0f..8cf9a6ffd7 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -29,7 +29,7 @@ frappe.breadcrumbs = { return localStorage["preferred_breadcrumbs:" + doctype]; }, - async add(module, doctype, type) { + add(module, doctype, type) { let obj; if (typeof module === "object") { obj = module; @@ -40,7 +40,6 @@ frappe.breadcrumbs = { type: type, }; } - await frappe.model.with_doctype(doctype); this.all[frappe.breadcrumbs.current_page()] = obj; this.update(); }, @@ -132,8 +131,9 @@ frappe.breadcrumbs = { } }, - set_list_breadcrumb(breadcrumbs) { + async set_list_breadcrumb(breadcrumbs) { const doctype = breadcrumbs.doctype; + await frappe.model.with_doctype(doctype); const doctype_meta = frappe.get_doc("DocType", doctype); if ( (doctype === "User" && !frappe.user.has_role("System Manager")) || From b71d93ef9f700a6a70a21df59314ed91c9647659 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 9 Nov 2022 14:26:06 +0530 Subject: [PATCH 125/167] test: db timeout computation --- frappe/tests/test_db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 9a7d086252..d8259975da 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -11,13 +11,13 @@ import frappe from frappe.core.utils import find from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.database import savepoint -from frappe.database.database import Database +from frappe.database.database import Database, get_query_execution_timeout from frappe.database.utils import FallBackDateTimeStr from frappe.query_builder import Field from frappe.query_builder.functions import Concat_ws from frappe.tests.test_query_builder import db_type_is, run_only_if from frappe.tests.utils import FrappeTestCase -from frappe.utils import add_days, cint, now, random_string +from frappe.utils import add_days, cint, now, random_string, set_request from frappe.utils.testutils import clear_custom_fields @@ -57,6 +57,13 @@ class TestDB(FrappeTestCase): frappe.db.rollback(save_point=savepoint) self.fail("Long running queries not timing out") + @patch.dict(frappe.conf, {"http_timeout": 20, "enable_db_statement_timeout": 1}) + def test_db_timeout_computation(self): + set_request(method="GET", path="/") + self.assertEqual(get_query_execution_timeout(), 30) + frappe.local.request = None + self.assertEqual(get_query_execution_timeout(), 0) + def test_get_value(self): self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator") From b0cb1adc013b58fe025faf90cb2e07297201211e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 9 Nov 2022 15:02:03 +0530 Subject: [PATCH 126/167] ci: config cleanup and bump coverage --- .flake8 | 55 +++++++++++++++++++++++----- .github/helper/flake8.conf | 75 -------------------------------------- .pre-commit-config.yaml | 1 - .stylelintrc | 9 ----- bandit.yml | 1 - pyproject.toml | 2 +- 6 files changed, 47 insertions(+), 96 deletions(-) delete mode 100644 .github/helper/flake8.conf delete mode 100644 .stylelintrc delete mode 100644 bandit.yml diff --git a/.flake8 b/.flake8 index 4b852abd7c..2de7a154c9 100644 --- a/.flake8 +++ b/.flake8 @@ -1,37 +1,74 @@ [flake8] ignore = + B001, + B007, + B009, + B010, + B950, + E101, + E111, + E114, + E116, + E117, E121, + E122, + E123, + E124, + E125, E126, E127, E128, + E131, + E201, + E202, E203, + E211, + E221, + E222, + E223, + E224, E225, E226, + E228, E231, E241, + E242, E251, E261, + E262, E265, + E266, + E271, + E272, + E273, + E274, + E301, E302, E303, E305, + E306, E402, E501, + E502, + E701, + E702, + E703, E741, + F401, + F403, + F405, + W191, W291, W292, W293, W391, W503, W504, - F403, - B007, - B950, - W191, - E124, # closing bracket, irritating while writing QB code - E131, # continuation line unaligned for hanging indent - E123, # closing bracket does not match indentation of opening bracket's line - E101, # ensured by use of black + E711, + E129, + F841, + E713, + E712, max-line-length = 200 -exclude=.github/helper/semgrep_rules +exclude=,test_*.py diff --git a/.github/helper/flake8.conf b/.github/helper/flake8.conf deleted file mode 100644 index 20d4b912ca..0000000000 --- a/.github/helper/flake8.conf +++ /dev/null @@ -1,75 +0,0 @@ -[flake8] -ignore = - B001, - B007, - B009, - B010, - B950, - E101, - E111, - E114, - E116, - E117, - E121, - E122, - E123, - E124, - E125, - E126, - E127, - E128, - E131, - E201, - E202, - E203, - E211, - E221, - E222, - E223, - E224, - E225, - E226, - E228, - E231, - E241, - E242, - E251, - E261, - E262, - E265, - E266, - E271, - E272, - E273, - E274, - E301, - E302, - E303, - E305, - E306, - E402, - E501, - E502, - E701, - E702, - E703, - E741, - F401, - F403, - F405, - W191, - W291, - W292, - W293, - W391, - W503, - W504, - E711, - E129, - F841, - E713, - E712, - - -max-line-length = 200 -exclude=.github/helper/semgrep_rules,test_*.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27fae671c9..0783e94457 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,7 +59,6 @@ repos: hooks: - id: flake8 additional_dependencies: ['flake8-bugbear',] - args: ['--config', '.github/helper/flake8.conf'] ci: autoupdate_schedule: weekly diff --git a/.stylelintrc b/.stylelintrc deleted file mode 100644 index 1e05d1fb41..0000000000 --- a/.stylelintrc +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": ["stylelint-config-recommended"], - "plugins": ["stylelint-scss"], - "rules": { - "at-rule-no-unknown": null, - "scss/at-rule-no-unknown": true, - "no-descending-specificity": null - } -} \ No newline at end of file diff --git a/bandit.yml b/bandit.yml deleted file mode 100644 index b8560e97c8..0000000000 --- a/bandit.yml +++ /dev/null @@ -1 +0,0 @@ -skips: ['E0203', 'B605', 'B404', 'B603', 'B607'] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 348353003c..6506bfa13c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ ensure_newline_before_comments = true indent = "\t" [tool.bench.dev-dependencies] -coverage = "~=6.4.1" +coverage = "~=6.5.0" Faker = "~=13.12.1" pyngrok = "~=5.0.5" unittest-xml-reporting = "~=3.0.4" From ffdd368fe298f7245070d5d6a362197dc9ef1f05 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 9 Nov 2022 14:08:28 +0530 Subject: [PATCH 127/167] perf: faster as_list and as_dict --- frappe/database/database.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 1934bc8c30..8f8aa2bba2 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -7,6 +7,7 @@ import random import re import string import traceback +import warnings from contextlib import contextmanager, suppress from time import time @@ -273,6 +274,20 @@ class Database: if pluck: return [r[0] for r in self.last_result] + if as_utf8: + warnings.warn( + "as_utf8 parameter is deprecated and will be removed in version 15.", + DeprecationWarning, + stacklevel=2, + ) + + if formatted: + warnings.warn( + "formatted parameter is deprecated and will be removed in version 15.", + DeprecationWarning, + stacklevel=2, + ) + # scrub output if required if as_dict: ret = self.fetch_as_dict(formatted, as_utf8) @@ -391,10 +406,13 @@ class Database: def fetch_as_dict(self, formatted=0, as_utf8=0) -> list[frappe._dict]: """Internal. Converts results to dict.""" result = self.last_result - ret = [] if result: keys = [column[0] for column in self._cursor.description] + if not as_utf8: + return [frappe._dict(zip(keys, row)) for row in result] + + ret = [] for r in result: values = [] for value in r: @@ -429,6 +447,9 @@ class Database: @staticmethod def convert_to_lists(res, formatted=0, as_utf8=0): """Convert tuple output to lists (internal).""" + if not as_utf8: + return [[value for value in row] for row in res] + nres = [] for r in res: nr = [] From 66f5f4dd467087a7a4eeb19a9a6566201550ee89 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 9 Nov 2022 14:28:52 +0530 Subject: [PATCH 128/167] refactor(db): deprecated unused functions --- frappe/database/database.py | 22 ++++++++++------------ frappe/utils/deprecations.py | 27 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 frappe/utils/deprecations.py diff --git a/frappe/database/database.py b/frappe/database/database.py index 8f8aa2bba2..91f73ba006 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -7,7 +7,6 @@ import random import re import string import traceback -import warnings from contextlib import contextmanager, suppress from time import time @@ -31,6 +30,7 @@ from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.utils import cast as cast_fieldtype from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool +from frappe.utils.deprecations import deprecated, deprecation_warning IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") @@ -275,18 +275,9 @@ class Database: return [r[0] for r in self.last_result] if as_utf8: - warnings.warn( - "as_utf8 parameter is deprecated and will be removed in version 15.", - DeprecationWarning, - stacklevel=2, - ) - + deprecation_warning("as_utf8 parameter is deprecated and will be removed in version 15.") if formatted: - warnings.warn( - "formatted parameter is deprecated and will be removed in version 15.", - DeprecationWarning, - stacklevel=2, - ) + deprecation_warning("formatted parameter is deprecated and will be removed in version 15.") # scrub output if required if as_dict: @@ -858,6 +849,7 @@ class Database: ).run(debug=debug, run=run, as_dict=as_dict) return {} + @deprecated def update(self, *args, **kwargs): """Update multiple values. Alias for `set_value`.""" return self.set_value(*args, **kwargs) @@ -897,6 +889,9 @@ class Database: modified_by = modified_by or frappe.session.user to_update.update({"modified": modified, "modified_by": modified_by}) + if for_update: + deprecation_warning("for_update parameter is deprecated and will be removed in v15.") + if is_single_doctype: frappe.db.delete( "Singles", filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug @@ -927,11 +922,13 @@ class Database: if dt in self.value_cache: del self.value_cache[dt] + @deprecated @staticmethod def set(doc, field, val): """Set value in document. **Avoid**""" doc.db_set(field, val) + @deprecated def touch(self, doctype, docname): """Update the modified timestamp of this document.""" modified = now() @@ -1254,6 +1251,7 @@ class Database: """ return self.sql_ddl(f"truncate `{get_table_name(doctype)}`") + @deprecated def clear_table(self, doctype): return self.truncate(doctype) diff --git a/frappe/utils/deprecations.py b/frappe/utils/deprecations.py new file mode 100644 index 0000000000..e98362198b --- /dev/null +++ b/frappe/utils/deprecations.py @@ -0,0 +1,27 @@ +""" Utils for deprecating functionality in Framework. + +WARNING: This file is internal, instead of depending just copy the code or use deprecation +libraries. +""" +import functools +import warnings + + +def deprecated(func): + """Decorator to wrap a function/method as deprecated.""" + + @functools.wraps(func) + def wrapper(*args, **kwargs): + deprecation_warning( + f"{func.__name__} is deprecated and will be removed in next major version.", + stacklevel=1, + ) + return func(*args, **kwargs) + + return wrapper + + +def deprecation_warning(message, category=DeprecationWarning, stacklevel=1): + """like warnings.warn but with auto incremented sane stacklevel.""" + + warnings.warn(message=message, category=category, stacklevel=stacklevel + 2) From a5240824d5cf25102d37ef3cc78d3045a11c8b3a Mon Sep 17 00:00:00 2001 From: Himanshu Shivhare Date: Wed, 9 Nov 2022 18:25:00 +0530 Subject: [PATCH 129/167] docs: youtube channel in "about" section (#18810) * YouTube channel addded in about section. I have added ERPNext YouTube channel reference in the about section. #Create an official Handle for YouTube Channel (Like @erpnext or erpnextofficial) * chore: handle [skip ci] Co-authored-by: Ankush Menat --- frappe/public/js/frappe/ui/toolbar/about.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/public/js/frappe/ui/toolbar/about.js b/frappe/public/js/frappe/ui/toolbar/about.js index e23706eff1..69cbbfaba0 100644 --- a/frappe/public/js/frappe/ui/toolbar/about.js +++ b/frappe/public/js/frappe/ui/toolbar/about.js @@ -19,6 +19,8 @@ frappe.ui.misc.about = function () { Facebook: https://facebook.com/erpnext

Twitter: https://twitter.com/erpnext

+

+ YouTube: https://www.youtube.com/@erpnextofficial

${__("Installed Apps")}

${__("Loading versions...")}
From 1dd719b123d08e8cc1dbc856ca234abe274772f9 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 9 Nov 2022 19:37:35 +0530 Subject: [PATCH 130/167] fix: decorator ordering statcimethod doesn't return normal function so can't be "chained" the other way around [skip ci] --- frappe/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 91f73ba006..47ca451289 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -922,8 +922,8 @@ class Database: if dt in self.value_cache: del self.value_cache[dt] - @deprecated @staticmethod + @deprecated def set(doc, field, val): """Set value in document. **Avoid**""" doc.db_set(field, val) From a42ca7d8c1d66fd9fd90fbcbeffac5281ea58b0b Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 9 Nov 2022 14:15:41 +0000 Subject: [PATCH 131/167] fix: raise exception if doc before save is not found (#18796) * fix: raise exception if doc before save is not found * test: ensure error is raised when trying to save new doc using `doc.save()` * chore: add comment explaining condition * test: clearer name and docstring --- frappe/model/document.py | 12 ++++++++---- frappe/tests/test_document.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 8dcc57e827..942c7005a2 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -744,12 +744,13 @@ class Document(BaseDocument): Will also validate document transitions (Save > Submit > Cancel) calling `self.check_docstatus_transition`.""" - self.load_doc_before_save() + self.load_doc_before_save(raise_exception=True) self._action = "save" - previous = self.get_doc_before_save() + previous = self._doc_before_save - if not previous or self.meta.get("is_virtual"): + # previous is None for new document insert + if not previous: self.check_docstatus_transition(0) return @@ -1048,7 +1049,7 @@ class Document(BaseDocument): self.set_title_field() - def load_doc_before_save(self): + def load_doc_before_save(self, *, raise_exception: bool = False): """load existing document from db before saving""" self._doc_before_save = None @@ -1059,6 +1060,9 @@ class Document(BaseDocument): try: self._doc_before_save = frappe.get_doc(self.doctype, self.name, for_update=True) except frappe.DoesNotExistError: + if raise_exception: + raise + frappe.clear_last_message() def run_post_save_methods(self): diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index cec7ee3bb0..3833e911a7 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -411,6 +411,19 @@ class TestDocument(FrappeTestCase): todo.save() self.assertEqual(todo.notify_update.call_count, 1) + def test_error_on_saving_new_doc_with_name(self): + """Trying to save a new doc with name should raise DoesNotExistError""" + + doc = frappe.get_doc( + { + "doctype": "ToDo", + "description": "this should raise frappe.DoesNotExistError", + "name": "lets-trick-doc-save", + } + ) + + self.assertRaises(frappe.DoesNotExistError, doc.save) + class TestDocumentWebView(FrappeTestCase): def get(self, path, user="Guest"): From 104942ee0a56809552992d37a8fa323e0ef16c0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Nov 2022 11:25:44 +0530 Subject: [PATCH 132/167] build(deps): bump socket.io-parser from 4.0.4 to 4.0.5 (#18823) Bumps [socket.io-parser](https://github.com/socketio/socket.io-parser) from 4.0.4 to 4.0.5. - [Release notes](https://github.com/socketio/socket.io-parser/releases) - [Changelog](https://github.com/socketio/socket.io-parser/blob/main/CHANGELOG.md) - [Commits](https://github.com/socketio/socket.io-parser/compare/4.0.4...4.0.5) --- updated-dependencies: - dependency-name: socket.io-parser dependency-type: indirect ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 263c531ca4..798300ae61 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3170,9 +3170,9 @@ socket.io-client@^4.5.1: socket.io-parser "~4.2.0" socket.io-parser@~4.0.4: - version "4.0.4" - resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.0.4.tgz#9ea21b0d61508d18196ef04a2c6b9ab630f4c2b0" - integrity sha512-t+b0SS+IxG7Rxzda2EVvyBZbvFPBCjJoyHuE0P//7OAsN23GItzDRdWa6ALxZI/8R5ygK7jAR6t028/z+7295g== + version "4.0.5" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.0.5.tgz#cb404382c32324cc962f27f3a44058cf6e0552df" + integrity sha512-sNjbT9dX63nqUFIOv95tTVm6elyIU4RvB1m8dOeZt+IgWwcWklFDOdmGcfo3zSiRsnR/3pJkjY5lfoGqEe4Eig== dependencies: "@types/component-emitter" "^1.2.10" component-emitter "~1.3.0" From fcaa16bb213702355449a2bc3aeda5bd1ff0b827 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 9 Nov 2022 22:25:47 +0530 Subject: [PATCH 133/167] perf: faster generate_hash --- frappe/__init__.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index c03b87be1c..483316a15f 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1018,19 +1018,12 @@ def get_precision( return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency) -def generate_hash(txt: str | None = None, length: int | None = None) -> str: - """Generates random hash for given text + current timestamp + random string.""" - import hashlib - import time +def generate_hash(txt: str | None = None, length: int | None = 56) -> str: + """Generate random hash using best available randomness source.""" + import math + import secrets - from .utils import random_string - - digest = hashlib.sha224( - ((txt or "") + repr(time.time()) + repr(random_string(8))).encode() - ).hexdigest() - if length: - digest = digest[:length] - return digest + return secrets.token_hex(math.ceil(length / 2))[:length] def reset_metadata_version(): From f34f7030a3372eb8dc2677bedcdf0948ffb20145 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Nov 2022 11:34:06 +0530 Subject: [PATCH 134/167] refactor: remove `txt` param from generate_hash use --- frappe/__init__.py | 5 ++++- frappe/core/doctype/data_export/exporter.py | 2 +- frappe/core/doctype/data_import/test_importer.py | 2 +- frappe/model/naming.py | 2 +- frappe/patches/v11_0/remove_skip_for_doctype.py | 2 +- frappe/patches/v12_0/move_email_and_phone_to_child_table.py | 6 +++--- frappe/patches/v12_0/setup_tags.py | 2 +- frappe/templates/includes/navbar/navbar_items.html | 4 ++-- frappe/tests/test_modules.py | 2 +- frappe/utils/jinja_globals.py | 4 +--- frappe/utils/oauth.py | 2 +- frappe/website/doctype/website_theme/website_theme.py | 2 +- .../section_with_collapsible_content.html | 2 +- .../web_template/section_with_tabs/section_with_tabs.html | 4 ++-- frappe/website/web_template/slideshow/slideshow.html | 2 +- 15 files changed, 22 insertions(+), 21 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 483316a15f..84a27642a9 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1018,11 +1018,14 @@ def get_precision( return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency) -def generate_hash(txt: str | None = None, length: int | None = 56) -> str: +def generate_hash(txt: str | None = None, length: int = 56) -> str: """Generate random hash using best available randomness source.""" import math import secrets + if not length: + length = 56 + return secrets.token_hex(math.ceil(length / 2))[:length] diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index e3bf669630..611592531d 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -432,7 +432,7 @@ class DataExporter: row[_column_start_end.start + i + 1] = value def build_response_as_excel(self): - filename = frappe.generate_hash("", 10) + filename = frappe.generate_hash(length=10) with open(filename, "wb") as f: f.write(cstr(self.writer.getvalue()).encode("utf-8")) f = open(filename) diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index af8c711ab5..978f5792dd 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -97,7 +97,7 @@ class TestImporter(FrappeTestCase): def test_data_import_update(self): existing_doc = frappe.get_doc( doctype=doctype_name, - title=frappe.generate_hash(doctype_name, 8), + title=frappe.generate_hash(length=8), table_field_1=[{"child_title": "child title to update"}], ) existing_doc.save() diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 6f7db5cf9d..93be2204b4 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -278,7 +278,7 @@ def make_autoname(key="", doctype="", doc=""): DE/09/01/00001 where 09 is the year, 01 is the month and 00001 is the series """ if key == "hash": - return frappe.generate_hash(doctype, 10) + return frappe.generate_hash(length=10) series = NamingSeries(key) return series.generate_next_name(doc) diff --git a/frappe/patches/v11_0/remove_skip_for_doctype.py b/frappe/patches/v11_0/remove_skip_for_doctype.py index e7c1d71a0a..b3471ca4e8 100644 --- a/frappe/patches/v11_0/remove_skip_for_doctype.py +++ b/frappe/patches/v11_0/remove_skip_for_doctype.py @@ -60,7 +60,7 @@ def execute(): # Maintain sequence (name, user, allow, for_value, applicable_for, apply_to_all_doctypes, creation, modified) new_user_permissions_list.append( ( - frappe.generate_hash("", 10), + frappe.generate_hash(length=10), user_permission.user, user_permission.allow, user_permission.for_value, diff --git a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py index 1a369b4e12..7283760c23 100644 --- a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py +++ b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py @@ -27,7 +27,7 @@ def execute(): email_values.append( ( 1, - frappe.generate_hash(contact_detail.email_id, 10), + frappe.generate_hash(length=10), contact_detail.email_id, "email_ids", "Contact", @@ -44,7 +44,7 @@ def execute(): phone_values.append( ( phone_counter, - frappe.generate_hash(contact_detail.email_id, 10), + frappe.generate_hash(length=10), contact_detail.phone, "phone_nos", "Contact", @@ -63,7 +63,7 @@ def execute(): phone_values.append( ( phone_counter, - frappe.generate_hash(contact_detail.email_id, 10), + frappe.generate_hash(length=10), contact_detail.mobile_no, "phone_nos", "Contact", diff --git a/frappe/patches/v12_0/setup_tags.py b/frappe/patches/v12_0/setup_tags.py index 6bff8d3dac..cb0d46a45d 100644 --- a/frappe/patches/v12_0/setup_tags.py +++ b/frappe/patches/v12_0/setup_tags.py @@ -28,7 +28,7 @@ def execute(): tag_list.append((tag.strip(), time, time, "Administrator")) - tag_link_name = frappe.generate_hash(_user_tags.name + tag.strip() + doctype.name, 10) + tag_link_name = frappe.generate_hash(length=10) tag_links.append( (tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, "Administrator") ) diff --git a/frappe/templates/includes/navbar/navbar_items.html b/frappe/templates/includes/navbar/navbar_items.html index 8a10751441..99a40a02a0 100644 --- a/frappe/templates/includes/navbar/navbar_items.html +++ b/frappe/templates/includes/navbar/navbar_items.html @@ -3,7 +3,7 @@ {% if parent %} -{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%} +{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
    • here" - + f". Execution time {round(float(time_diff), 2)} seconds", + + f". View it here", "alert": True, "indicator": "red" if submission_status == "Failed" else "green", }, @@ -128,8 +130,13 @@ class SubmissionQueue(Document): def _unlock_reference_doc(self): job_id = frappe.db.get_value(self.doctype, self.name, "job_id") if not job_id: - # assuming the job failed here (?) + # Assuming the job failed here (?) + # Or could be in Queue also since we are setting the job id during job.perform() + # No way to tell status = "failed" + else: + status = Job.fetch(job_id, connection=get_redis_conn()).get_status(refresh=True) + queued_doc = self.queued_doc # Job finished successfully however action was never completed (?) From e8b3f8f7106c79ff0e93f300b62ddb956243c523 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 08:57:48 +0530 Subject: [PATCH 151/167] refactor: only showing unlock button if job id is defined --- .../submission_queue/submission_queue.js | 2 +- .../submission_queue/submission_queue.py | 41 +++++-------------- 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js index 414c8c9ee0..93d6b981dc 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.js +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -3,7 +3,7 @@ frappe.ui.form.on("Submission Queue", { refresh: function (frm) { - if (frm.doc.status === "Queued") { + if (frm.doc.status === "Queued" && frm.doc.job_id) { frm.add_custom_button(__("Unlock Reference Document"), () => { frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => { frm.call("unlock_doc"); diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 00c6c86582..f8496d9aa8 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -3,7 +3,6 @@ from rq import get_current_job from rq.job import Job -from rq.registry import FailedJobRegistry import frappe from frappe import _ @@ -11,7 +10,7 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create from frappe.model.document import Document from frappe.monitor import add_data_to_monitor from frappe.utils import now, time_diff_in_seconds -from frappe.utils.background_jobs import get_queue, get_redis_conn +from frappe.utils.background_jobs import get_redis_conn from frappe.utils.data import cint @@ -128,37 +127,17 @@ class SubmissionQueue(Document): enqueue_create_notification([notify_to], notification_doc) def _unlock_reference_doc(self): - job_id = frappe.db.get_value(self.doctype, self.name, "job_id") - if not job_id: - # Assuming the job failed here (?) - # Or could be in Queue also since we are setting the job id during job.perform() - # No way to tell - status = "failed" - else: - status = Job.fetch(job_id, connection=get_redis_conn()).get_status(refresh=True) + """ + Only execute if self.job_id is defined. + """ + job = Job.fetch(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) - queued_doc = self.queued_doc - - # Job finished successfully however action was never completed (?) - if status == "finished" and queued_doc.docstatus == 0: - status = "failed" - - # Checking if job is queue to be executed/executing if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) - - # Checking any one of the possible termination statuses - elif status in ("failed", "canceled", "stopped"): - queued_doc.unlock() - values = {"status": "Failed"} - - # Defining job exception when unlocking document. - registry = FailedJobRegistry(queue=get_queue(qtype="default")) - # If job id is None then this job won't exist in the failed registry - for jid in registry.get_job_ids(): - if jid == job_id: - job = Job.fetch(job_id, connection=get_redis_conn()) - values = {"status": "Failed", "exception": job.exc_info} + else: + self.queued_doc.unlock() + values = {"status": "Failed", "exception": job.exc_info} frappe.db.set_value(self.doctype, self.name, values, update_modified=False) frappe.msgprint(_("Document Unlocked")) @@ -169,7 +148,7 @@ class SubmissionQueue(Document): # for example: hitting unlock on a submission could lead to unlocking of another submission # of the same reference document. - if self.status != "Queued": + if (self.status != "Queued") or (not self.job_id): return self._unlock_reference_doc() From 36033d41455c4f1e388ce087b12e6d2db9a6145b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 09:32:49 +0530 Subject: [PATCH 152/167] feat: Added finished condition while unlocking --- frappe/core/doctype/submission_queue/submission_queue.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index f8496d9aa8..b915e25d7d 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -135,6 +135,10 @@ class SubmissionQueue(Document): if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) + elif status == "finished": + self.queued_doc.unlock() + frappe.db.set_value(self.doctype, self.name, {"status": "Finished"}, update_modified=False) + frappe.msgprint(_("Document Unlocked")) else: self.queued_doc.unlock() values = {"status": "Failed", "exception": job.exc_info} From 198bc4275fc46838333e195101f2baa2ab5f830d Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sat, 12 Nov 2022 12:48:13 +0530 Subject: [PATCH 153/167] chore: fully commented, consistently formatted JS boilerplates --- frappe/core/doctype/doctype/boilerplate/controller.js | 8 ++++---- .../core/doctype/doctype/boilerplate/controller_list.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/doctype/boilerplate/controller.js b/frappe/core/doctype/doctype/boilerplate/controller.js index 6d9fb2a514..0e3dcd2e26 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller.js +++ b/frappe/core/doctype/doctype/boilerplate/controller.js @@ -1,8 +1,8 @@ // Copyright (c) {year}, {app_publisher} and contributors // For license information, please see license.txt -frappe.ui.form.on('{doctype}', {{ - // refresh: function(frm) {{ +// frappe.ui.form.on("{doctype}", {{ +// refresh(frm) {{ - // }} -}}); +// }}, +// }}); diff --git a/frappe/core/doctype/doctype/boilerplate/controller_list.js b/frappe/core/doctype/doctype/boilerplate/controller_list.js index b1f6d12008..3740cfa85d 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller_list.js +++ b/frappe/core/doctype/doctype/boilerplate/controller_list.js @@ -1,5 +1,5 @@ /* eslint-disable */ -frappe.listview_settings['{doctype}'] = {{ - // add_fields: ["status"], - // filters:[["status","=", "Open"]] -}}; +// frappe.listview_settings["{doctype}"] = {{ +// add_fields: ["status"], +// filters: [["status","=", "Open"]], +// }}; From 6c01d1d417cbf838e75add091a42bcaa6f5ce48e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 12 Nov 2022 12:02:04 +0530 Subject: [PATCH 154/167] refactor: hmac generation Reduce code duplication --- frappe/tests/test_email.py | 14 ++++++++++++++ frappe/utils/verified_command.py | 17 +++++++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index 13e95c38e6..de0fe00012 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -310,6 +310,20 @@ class TestEmail(FrappeTestCase): email_account.enable_incoming = False +class TestVerifiedRequests(FrappeTestCase): + def test_round_trip(self): + from frappe.utils import set_request + from frappe.utils.verified_command import get_signed_params, verify_request + + test_cases = [{"xyz": "abc"}, {"email": "a@b.com", "user": "xyz"}] + + for params in test_cases: + signed_url = get_signed_params(params) + set_request(method="GET", path="?" + signed_url) + self.assertTrue(verify_request()) + frappe.local.request = None + + if __name__ == "__main__": import unittest diff --git a/frappe/utils/verified_command.py b/frappe/utils/verified_command.py index b25e646cf5..3d6346276d 100644 --- a/frappe/utils/verified_command.py +++ b/frappe/utils/verified_command.py @@ -16,9 +16,8 @@ def get_signed_params(params): if not isinstance(params, str): params = urlencode(params) - signature = hmac.new(params.encode(), digestmod=hashlib.sha512) - signature.update(get_secret().encode()) - return params + "&_signature=" + signature.hexdigest() + signature = _sign_message(params) + return params + "&_signature=" + signature def get_secret(): @@ -35,12 +34,10 @@ def verify_request(): signature_string = "&_signature=" if signature_string in query_string: - params, signature = query_string.split(signature_string) + params, given_signature = query_string.split(signature_string) - given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.sha512) - - given_signature.update(get_secret().encode()) - valid_signature = hmac.compare_digest(signature, given_signature.hexdigest()) + computed_signature = _sign_message(params) + valid_signature = hmac.compare_digest(given_signature, computed_signature) valid_method = frappe.request.method == "GET" valid_request_data = not (frappe.request.form or frappe.request.data) @@ -55,6 +52,10 @@ def verify_request(): return False +def _sign_message(message: str) -> str: + return hmac.new(get_secret().encode(), message.encode(), digestmod=hashlib.sha512).hexdigest() + + def get_url(cmd, params, nonce=None, secret=None): if not nonce: nonce = params From 723a27bda777c3e9ddcab5b84a52155ac6f2ddb5 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:08:29 +0530 Subject: [PATCH 155/167] fix: try except for nosuchjoberror in _unlock_reference_doc * fix: condition for not unlocking reference doc * chore: change id param -> job_id for update_job_id --- .../submission_queue/submission_queue.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b915e25d7d..dcb0548e4e 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -2,6 +2,7 @@ # For license information, please see license.txt from rq import get_current_job +from rq.exceptions import NoSuchJobError from rq.job import Job import frappe @@ -46,11 +47,11 @@ class SubmissionQueue(Document): def unlock(self): self.queued_doc.unlock() - def update_job_id(self, id): + def update_job_id(self, job_id): frappe.db.set_value( self.doctype, self.name, - {"job_id": id}, + {"job_id": job_id}, update_modified=False, ) frappe.db.commit() @@ -65,9 +66,9 @@ class SubmissionQueue(Document): ) def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): - current_job = get_current_job() # Set the job id for that submission doctype - self.update_job_id(current_job.id) + self.update_job_id(get_current_job().id) + _action = action_for_queuing.lower() if _action == "update": _action = "submit" @@ -130,21 +131,25 @@ class SubmissionQueue(Document): """ Only execute if self.job_id is defined. """ - job = Job.fetch(self.job_id, connection=get_redis_conn()) - status = job.get_status(refresh=True) + try: + job = Job.fetch(self.job_id, connection=get_redis_conn()) + status = job.get_status(refresh=True) + except NoSuchJobError: + # assuming job failed over here (?) + status = "failed" if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) - elif status == "finished": - self.queued_doc.unlock() - frappe.db.set_value(self.doctype, self.name, {"status": "Finished"}, update_modified=False) - frappe.msgprint(_("Document Unlocked")) - else: - self.queued_doc.unlock() - values = {"status": "Failed", "exception": job.exc_info} + return - frappe.db.set_value(self.doctype, self.name, values, update_modified=False) - frappe.msgprint(_("Document Unlocked")) + self.queued_doc.unlock() + values = ( + {"status": "Finished"} + if status == "finished" + else {"status": "Failed", "exception": job.exc_info} + ) + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + frappe.msgprint(_("Document Unlocked")) @frappe.whitelist() def unlock_doc(self): @@ -152,7 +157,7 @@ class SubmissionQueue(Document): # for example: hitting unlock on a submission could lead to unlocking of another submission # of the same reference document. - if (self.status != "Queued") or (not self.job_id): + if self.status != "Queued" and not self.job_id: return self._unlock_reference_doc() From b21f9a5b264116fbba42acc598ae91ee0f09e91d Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:10:46 +0530 Subject: [PATCH 156/167] fix: submission queue banner if last_failed_submission and last_submissions are equal, show only last_failed_submission --- .../submission_queue/submission_queue.py | 2 +- frappe/public/js/frappe/form/form.js | 32 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index dcb0548e4e..5e400e8707 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -188,5 +188,5 @@ def get_latest_submissions(doctype, docname): filters = {"ref_doctype": doctype, "ref_docname": docname} return { "latest_submission": frappe.db.get_value(dt, filters), - "latest_failed_submission": frappe.db.get_value(dt, filters.update({"status": "Failed"})), + "latest_failed_submission": frappe.db.get_value(dt, filters | {"status": "Failed"}), } diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8027f32454..fa27eaaf9f 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -2073,7 +2073,7 @@ frappe.ui.form.Form = class FrappeForm { } if (!wrapper.length) { - wrapper = $('
    '); + wrapper = $('
    '); this.layout.wrapper.prepend(wrapper); } @@ -2085,16 +2085,21 @@ frappe.ui.form.Form = class FrappeForm { .then((r) => { if (r.message.latest_submission) { // if we are here that means some submission(s) were queued and are in queued/failed state - wrapper.show(); let col_width = 4; let failed_link = ""; + let submission_label = __("Previous Submission"); + if (r.message.latest_failed_submission) { - col_width = 3; - failed_link = `
    - ${__( - "Previous Failed Submission" - )} -
    `; + if (r.message.latest_failed_submission !== r.message.latest_submission) { + col_width = 3; + failed_link = `
    + ${__( + "Previous Falied Submission" + )} +
    `; + } else { + submission_label = __("Previous Falied Submission"); + } } let html = ` @@ -2103,17 +2108,20 @@ frappe.ui.form.Form = class FrappeForm { ${__("Submission Status:")}
    - ${__("Previous Submission")} + ${submission_label}
    ${failed_link}
    - ${__( - "All Submissions" - )} + ${__( + "All Submissions" + )}
    `; + wrapper.show(); wrapper.html(html); } else { wrapper.hide(); From 3dc376cd5ccc27f2930fc39ee712ac9e647c5766 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:16:05 +0530 Subject: [PATCH 157/167] chore: remove reload listener from form --- frappe/public/js/frappe/form/form.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index fa27eaaf9f..4f119f2551 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -397,8 +397,6 @@ frappe.ui.form.Form = class FrappeForm { // set the doc this.doc = frappe.get_doc(this.doctype, this.docname); - if (!this.doc.__islocal) this.setup_reload_listener(); - // check permissions this.fetch_permissions(); if (!this.has_read_permission()) { @@ -1979,19 +1977,6 @@ frappe.ui.form.Form = class FrappeForm { }); } - setup_reload_listener() { - let doctype = this.doctype; - let docname = this.docname; - let listener_name = `reload_doc_${doctype}_${docname}`; - - frappe.realtime.off(listener_name); - frappe.realtime.on(listener_name, () => { - if (frappe.get_route_str() === `Form/${doctype}/${docname}`) { - this.reload_doc(); - } - }); - } - // Filters fields from the reference doctype and sets them as options for a Select field set_fields_as_options( fieldname, From ad6a11e34c4e6b248905512487ccf1cba2081ccb Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 15:44:20 +0530 Subject: [PATCH 158/167] feat: queue submission for bulk submit --- .../submission_queue/submission_queue.py | 17 +++++++++-------- frappe/desk/doctype/bulk_update/bulk_update.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 5e400e8707..b7ed9cdcc2 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -163,20 +163,21 @@ class SubmissionQueue(Document): self._unlock_reference_doc() -def queue_submission(doc: Document, action: str): +def queue_submission(doc: Document, action: str, alert: bool = True): queue = frappe.new_doc("Submission Queue") queue.state = "Queued" queue.ref_doctype = doc.doctype queue.ref_docname = doc.name queue.insert(doc, action) - frappe.msgprint( - _("Queued for Submission. You can track the progress over {0}.").format( - f"here" - ), - indicator="green", - alert=True, - ) + if alert: + frappe.msgprint( + _("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) @frappe.whitelist() diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 1e515bbc47..5521d9583f 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -3,8 +3,10 @@ import frappe from frappe import _ +from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.model.document import Document from frappe.utils import cint +from frappe.utils.scheduler import is_scheduler_inactive class BulkUpdate(Document): @@ -44,8 +46,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): try: message = "" if action == "submit" and doc.docstatus.is_draft(): - doc.submit() - message = _("Submitting {0}").format(doctype) + if doc.meta.queue_in_background and not is_scheduler_inactive(): + queue_submission(doc, action) + message = _("Queuing {0} for Submission").format(doctype) + else: + doc.submit() + message = _("Submitting {0}").format(doctype) elif action == "cancel" and doc.docstatus.is_submitted(): doc.cancel() message = _("Cancelling {0}").format(doctype) From f9a10d32d10fc62231010a7178a8a2992628bf9b Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 16:19:44 +0530 Subject: [PATCH 159/167] fix: use quoted doctype and docname in alert --- frappe/core/doctype/submission_queue/submission_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index b7ed9cdcc2..6c0ad70bd9 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -1,6 +1,8 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +from urllib.parse import quote + from rq import get_current_job from rq.exceptions import NoSuchJobError from rq.job import Job @@ -110,7 +112,7 @@ class SubmissionQueue(Document): "msgprint", { "message": message - + f". View it here", + + f". View it here", "alert": True, "indicator": "red" if submission_status == "Failed" else "green", }, From b09eb2f317de1982fd25565040f09482db8e39ba Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 12 Nov 2022 16:51:58 +0530 Subject: [PATCH 160/167] fix: use index for ref_docname in submission queue doctype --- frappe/core/doctype/submission_queue/submission_queue.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json index 7d2a1e0703..d1f66ffa13 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.json +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -38,7 +38,8 @@ "in_list_view": 1, "label": "Reference Docname", "options": "ref_doctype", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "fieldname": "status", @@ -86,7 +87,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2022-11-09 01:01:27.185383", + "modified": "2022-11-12 16:48:37.797232", "modified_by": "Administrator", "module": "Core", "name": "Submission Queue", From 1145bab9d89891bac6e3f5031d15410f336ecf0c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 19:24:41 +0530 Subject: [PATCH 161/167] refactor: checking docstatus before asserting failure --- .../doctype/submission_queue/submission_queue.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 6c0ad70bd9..872284d46c 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -133,12 +133,18 @@ class SubmissionQueue(Document): """ Only execute if self.job_id is defined. """ + exc = None try: job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) + exc = job.exc_info except NoSuchJobError: - # assuming job failed over here (?) - status = "failed" + # Document is submitted. + if self.queued_doc.docstatus == 1: + status = "finished" + # Document is not submitted and job doesn't exist. + else: + status = "failed" if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) @@ -146,9 +152,7 @@ class SubmissionQueue(Document): self.queued_doc.unlock() values = ( - {"status": "Finished"} - if status == "finished" - else {"status": "Failed", "exception": job.exc_info} + {"status": "Finished"} if status == "finished" else {"status": "Failed", "exception": exc} ) frappe.db.set_value(self.doctype, self.name, values, update_modified=False) frappe.msgprint(_("Document Unlocked")) From 7c8af96046fa7bb2fa436b35d940885a20e0b215 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 19:39:47 +0530 Subject: [PATCH 162/167] refactor: leaving job status as it is on completion --- .../doctype/submission_queue/submission_queue.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 872284d46c..39f36fa356 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -70,6 +70,9 @@ class SubmissionQueue(Document): def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): # Set the job id for that submission doctype self.update_job_id(get_current_job().id) + import time + + time.sleep(10) _action = action_for_queuing.lower() if _action == "update": @@ -133,18 +136,13 @@ class SubmissionQueue(Document): """ Only execute if self.job_id is defined. """ - exc = None try: job = Job.fetch(self.job_id, connection=get_redis_conn()) status = job.get_status(refresh=True) exc = job.exc_info except NoSuchJobError: - # Document is submitted. - if self.queued_doc.docstatus == 1: - status = "finished" - # Document is not submitted and job doesn't exist. - else: - status = "failed" + exc = None + status = "failed" if status in ("queued", "started"): frappe.msgprint(_("Document in queue for execution!")) From 3759e5bbcd32dfe830b36ce8a5b76a7059f363e1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 12 Nov 2022 19:41:21 +0530 Subject: [PATCH 163/167] refactor: lint fix: removed time.sleep --- frappe/core/doctype/submission_queue/submission_queue.py | 4 ---- frappe/desk/form/save.py | 6 +----- frappe/tests/test_document_locks.py | 7 +++---- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py index 39f36fa356..2bb4200a87 100644 --- a/frappe/core/doctype/submission_queue/submission_queue.py +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -70,10 +70,6 @@ class SubmissionQueue(Document): def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): # Set the job id for that submission doctype self.update_job_id(get_current_job().id) - import time - - time.sleep(10) - _action = action_for_queuing.lower() if _action == "update": _action = "submit" diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 8ea5b120e6..f43031c899 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -19,11 +19,7 @@ def savedocs(doc, action): # action doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] if doc.docstatus == 1: - if ( - action == "Submit" - and doc.meta.queue_in_background - and not is_scheduler_inactive() - ): + if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive(): queue_submission(doc, action) return doc.submit() diff --git a/frappe/tests/test_document_locks.py b/frappe/tests/test_document_locks.py index 2ba606685e..5d19f75050 100644 --- a/frappe/tests/test_document_locks.py +++ b/frappe/tests/test_document_locks.py @@ -27,13 +27,12 @@ class TestDocumentLocks(FrappeTestCase): # Checking for persistant locks across all instances. doc = frappe.get_doc("ToDo", todo.name) - self.assertEquals(doc.is_locked, True) + self.assertEqual(doc.is_locked, True) with self.assertRaises(frappe.DocumentLockedError): doc.description = "Random" doc.save() doc.unlock() - self.assertEquals(doc.is_locked, False) - self.assertEquals(todo.is_locked, False) - + self.assertEqual(doc.is_locked, False) + self.assertEqual(todo.is_locked, False) From 838a52328cd3d0f49fafdbad7c955a5e53d5cfe3 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sat, 12 Nov 2022 23:51:02 +0530 Subject: [PATCH 164/167] fix: hardcode doctype in google oauth callback --- frappe/email/oauth.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 89b6df15d8..f5b60a9f3d 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -132,19 +132,18 @@ def oauth_access(email_account: str, service: str): if not service: frappe.throw(frappe._("No Service is selected. Please select one and try again!")) - doctype = "Email Account" - if service == "GMail": - return authorize_google_access(email_account, doctype) + return authorize_google_access(email_account) raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.") -def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None): +def authorize_google_access(email_account: str, code: str = None): """Facilitates google oauth for email. - This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url + This is invoked 2 times - first time when user clicks `Authorize API Access` for getting the authorization url and second time for setting the refresh and access token in db when google redirects back with oauth code.""" + doctype = "Email Account" oauth_obj = GoogleOAuth("mail") if not code: From f019b4fab610b9417b5ceda325aa954cd8375998 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 14 Nov 2022 15:46:54 +0530 Subject: [PATCH 165/167] build(deps): update caniuse (#18866) [skip ci] --- .github/workflows/ui-tests.yml | 1 - package.json | 24 ++++++++++++------------ yarn.lock | 18 ++++-------------- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 8a78d2e750..1a122a3b12 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -121,7 +121,6 @@ jobs: DB: mariadb - name: Verify yarn.lock - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | cd ~/frappe-bench/apps/frappe yarn install --immutable --immutable-cache --check-cache diff --git a/package.json b/package.json index f92deb16a7..5e6754292b 100644 --- a/package.json +++ b/package.json @@ -22,16 +22,24 @@ "homepage": "https://frappeframework.com", "dependencies": { "@editorjs/editorjs": "2.20.0", + "@frappe/esbuild-plugin-postcss2": "^0.1.3", + "@vue/component-compiler": "^4.2.4", "ace-builds": "^1.4.8", "air-datepicker": "github:frappe/air-datepicker", + "autoprefixer": "10", "awesomplete": "^1.1.5", "bootstrap": "4.5.0", + "chalk": "^2.3.2", + "cliui": "^7.0.4", "cookie": "^0.4.0", "cropperjs": "^1.5.12", "cssnano": "^5.0.0", "driver.js": "^0.9.8", "editorjs-undo": "0.1.6", + "esbuild": "^0.14.29", + "esbuild-plugin-vue3": "^0.3.0", "fast-deep-equal": "^2.0.1", + "fast-glob": "^3.2.5", "frappe-charts": "2.0.0-rc22", "frappe-datatable": "^1.16.4", "frappe-gantt": "^0.6.0", @@ -40,16 +48,20 @@ "jquery": "3.6.0", "js-sha256": "^0.9.0", "jsbarcode": "^3.9.0", + "launch-editor": "^2.2.1", "localforage": "^1.9.0", + "md5": "^2.3.0", "moment": "^2.29.4", "moment-timezone": "^0.5.35", "plyr": "^3.7.2", "popper.js": "^1.16.0", + "postcss": "8", "quill": "2.0.0-dev.4", "quill-image-resize": "^3.0.9", "quill-magic-url": "^3.0.0", "qz-tray": "^2.0.8", "redis": "^3.1.1", + "rtlcss": "^3.2.1", "sass": "^1.53.0", "showdown": "^2.1.0", "snyk": "^1.996.0", @@ -62,18 +74,6 @@ "vue-router": "^4.1.5", "vuedraggable": "^4.1.0", "vuex": "4.0.2", - "@frappe/esbuild-plugin-postcss2": "^0.1.3", - "@vue/component-compiler": "^4.2.4", - "autoprefixer": "10", - "chalk": "^2.3.2", - "cliui": "^7.0.4", - "esbuild": "^0.14.29", - "esbuild-plugin-vue3": "^0.3.0", - "fast-glob": "^3.2.5", - "launch-editor": "^2.2.1", - "md5": "^2.3.0", - "postcss": "8", - "rtlcss": "^3.2.1", "yargs": "^17.5.1" }, "snyk": true, diff --git a/yarn.lock b/yarn.lock index 798300ae61..dc0f79efaa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -532,20 +532,10 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001358: - version "1.0.30001359" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001359.tgz#a1c1cbe1c2da9e689638813618b4219acbd4925e" - integrity sha512-Xln/BAsPzEuiVLgJ2/45IaqD9jShtk3Y33anKb4+yLwQzws3+v6odKfpgES/cDEaZMLzSChpIGdbOYtH9MyuHw== - -caniuse-lite@^1.0.30001196: - version "1.0.30001296" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz" - integrity sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q== - -caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001313: - version "1.0.30001316" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001316.tgz#b44a1f419f82d2e119aa0bbdab5ec15471796358" - integrity sha512-JgUdNoZKxPZFzbzJwy4hDSyGuH/gXz2rN51QmoR8cBQsVo58llD3A0vlRKKRt8FGf5u69P9eQyIH8/z9vN/S0Q== +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001196, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001313, caniuse-lite@^1.0.30001358: + version "1.0.30001431" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001431.tgz" + integrity sha512-zBUoFU0ZcxpvSt9IU66dXVT/3ctO1cy4y9cscs1szkPlcWb6pasYM144GqrUygUbT+k7cmUCW61cvskjcv0enQ== chalk@^1.1.3: version "1.1.3" From feb9190cac2006f84535489c4a6e66782be94f55 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Mon, 14 Nov 2022 15:54:41 +0530 Subject: [PATCH 166/167] fix: check if the doctype exists before adding default logtypes in log settings (#18867) --- frappe/core/doctype/log_settings/log_settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index f1b5d23c4f..f0f2cdaae8 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -69,6 +69,9 @@ class LogSettings(Document): added_logtypes = set() for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items(): if logtype not in existing_logtypes and _supports_log_clearing(logtype): + if not frappe.db.exists("DocType", logtype): + continue + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)}) added_logtypes.add(logtype) From 990a96e48bc534a73f499dc47e3dd4897a8fae94 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 14 Nov 2022 15:55:20 +0530 Subject: [PATCH 167/167] feat: show utilization percent on RQ Worker (#18868) [skip ci] --- frappe/core/doctype/rq_worker/rq_worker.json | 12 +++++++++--- frappe/core/doctype/rq_worker/rq_worker.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/rq_worker/rq_worker.json b/frappe/core/doctype/rq_worker/rq_worker.json index ea65abd482..d9a5a23f67 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.json +++ b/frappe/core/doctype/rq_worker/rq_worker.json @@ -20,7 +20,8 @@ "column_break_12", "birth_date", "last_heartbeat", - "total_working_time" + "total_working_time", + "utilization_percent" ], "fields": [ { @@ -59,7 +60,6 @@ { "fieldname": "successful_job_count", "fieldtype": "Int", - "in_list_view": 1, "label": "Successful Job Count" }, { @@ -102,12 +102,18 @@ { "fieldname": "column_break_12", "fieldtype": "Column Break" + }, + { + "fieldname": "utilization_percent", + "fieldtype": "Percent", + "in_list_view": 1, + "label": "Utilization %" } ], "in_create": 1, "is_virtual": 1, "links": [], - "modified": "2022-09-11 05:02:53.981705", + "modified": "2022-11-14 15:35:32.786012", "modified_by": "Administrator", "module": "Core", "name": "RQ Worker", diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py index b2d1f1209d..3de0c8f7fc 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.py +++ b/frappe/core/doctype/rq_worker/rq_worker.py @@ -1,6 +1,9 @@ # Copyright (c) 2022, Frappe Technologies and contributors # For license information, please see license.txt +import datetime +from contextlib import suppress + from rq import Worker import frappe @@ -66,4 +69,11 @@ def serialize_worker(worker: Worker) -> frappe._dict: _comment_count=0, modified=convert_utc_to_user_timezone(worker.last_heartbeat), creation=convert_utc_to_user_timezone(worker.birth_date), + utilization_percent=compute_utilization(worker), ) + + +def compute_utilization(worker: Worker) -> float: + with suppress(Exception): + total_time = (datetime.datetime.utcnow() - worker.birth_date).total_seconds() + return worker.total_working_time / total_time * 100