From d6ab900f2a49a44360472dd3eaa026eb03973b4e Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Thu, 3 Aug 2023 11:55:19 +0530 Subject: [PATCH 01/24] feat: allow virtual child tables --- frappe/core/doctype/doctype/doctype.json | 1 - frappe/core/doctype/doctype/doctype.py | 14 +++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index d42fa62802..6cea37b539 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -538,7 +538,6 @@ }, { "default": "0", - "depends_on": "eval:!doc.istable", "fieldname": "is_virtual", "fieldtype": "Check", "label": "Is Virtual" diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index f7e6f28527..148e243ee3 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -923,7 +923,7 @@ class DocType(Document): self.nsm_parent_field = parent_field_name def validate_child_table(self): - if not self.get("istable") or self.is_new() or self.get("is_virtual"): + if not self.get("istable") or self.is_new(): # if the doctype is not a child table then return # if the doctype is a new doctype and also a child table then # don't move forward as it will be handled via schema @@ -1524,9 +1524,9 @@ def validate_fields(meta): return doctype = docfield.options - meta = frappe.get_meta(doctype) + child_doctype_meta = frappe.get_meta(doctype) - if not meta.istable: + if not child_doctype_meta.istable: frappe.throw( _("Option {0} for field {1} is not a child table").format( frappe.bold(doctype), frappe.bold(docfield.fieldname) @@ -1534,6 +1534,14 @@ def validate_fields(meta): title=_("Invalid Option"), ) + if not (meta.is_virtual == child_doctype_meta.is_virtual): + frappe.throw( + _( + "Option {0} for field {1} - Either both or none of the parent and child doctypes should be virtual" + ).format(frappe.bold(doctype), frappe.bold(docfield.fieldname)), + title=_("Invalid Option"), + ) + def check_max_height(docfield): if getattr(docfield, "max_height", None) and (docfield.max_height[-2:] not in ("px", "em")): frappe.throw(f"Max for {frappe.bold(docfield.fieldname)} height must be in px, em, rem") From 44c61453739146e56550d4119cea87312e5e569f Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Thu, 3 Aug 2023 12:03:30 +0530 Subject: [PATCH 02/24] feat: add child table for recorder requests --- .../core/doctype/recorder_request/__init__.py | 0 .../recorder_request/recorder_request.js | 6 + .../recorder_request/recorder_request.json | 123 ++++++++++ .../recorder_request/recorder_request.py | 211 ++++++++++++++++++ .../recorder_request/recorder_request_list.js | 47 ++++ .../recorder_request/test_recorder_request.py | 9 + 6 files changed, 396 insertions(+) create mode 100644 frappe/core/doctype/recorder_request/__init__.py create mode 100644 frappe/core/doctype/recorder_request/recorder_request.js create mode 100644 frappe/core/doctype/recorder_request/recorder_request.json create mode 100644 frappe/core/doctype/recorder_request/recorder_request.py create mode 100644 frappe/core/doctype/recorder_request/recorder_request_list.js create mode 100644 frappe/core/doctype/recorder_request/test_recorder_request.py diff --git a/frappe/core/doctype/recorder_request/__init__.py b/frappe/core/doctype/recorder_request/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/recorder_request/recorder_request.js b/frappe/core/doctype/recorder_request/recorder_request.js new file mode 100644 index 0000000000..59d813e2d7 --- /dev/null +++ b/frappe/core/doctype/recorder_request/recorder_request.js @@ -0,0 +1,6 @@ +// Copyright (c) 2023, Frappe Technologies and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Recorder Request", { + +// }); diff --git a/frappe/core/doctype/recorder_request/recorder_request.json b/frappe/core/doctype/recorder_request/recorder_request.json new file mode 100644 index 0000000000..9d46689bda --- /dev/null +++ b/frappe/core/doctype/recorder_request/recorder_request.json @@ -0,0 +1,123 @@ +{ + "actions": [], + "creation": "2023-08-01 12:06:49.630877", + "default_view": "List", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "path", + "number_of_queries", + "time_in_queries", + "method", + "column_break_qo53", + "cmd", + "time", + "duration", + "section_break_1skt", + "request_headers", + "section_break_sgro", + "form_dict", + "section_break_4umr", + "sql_queries" + ], + "fields": [ + { + "fieldname": "path", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Path" + }, + { + "fieldname": "cmd", + "fieldtype": "Data", + "label": "CMD" + }, + { + "fieldname": "duration", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Duration" + }, + { + "fieldname": "time", + "fieldtype": "Datetime", + "in_list_view": 1, + "label": "Time" + }, + { + "fieldname": "number_of_queries", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Number of Queries" + }, + { + "fieldname": "time_in_queries", + "fieldtype": "Float", + "label": "Time in Queries" + }, + { + "fieldname": "column_break_qo53", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_1skt", + "fieldtype": "Section Break" + }, + { + "fieldname": "request_headers", + "fieldtype": "Code", + "label": "Request Headers" + }, + { + "fieldname": "section_break_sgro", + "fieldtype": "Section Break" + }, + { + "fieldname": "form_dict", + "fieldtype": "Code", + "label": "Form Dict" + }, + { + "fieldname": "section_break_4umr", + "fieldtype": "Section Break" + }, + { + "fieldname": "sql_queries", + "fieldtype": "Table", + "label": "SQL Queries", + "options": "Recorder Query" + }, + { + "fieldname": "method", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Method" + } + ], + "hide_toolbar": 1, + "in_create": 1, + "index_web_pages_for_search": 1, + "is_virtual": 1, + "links": [], + "modified": "2023-08-02 20:36:12.191767", + "modified_by": "Administrator", + "module": "Core", + "name": "Recorder Request", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "path" +} \ No newline at end of file diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py new file mode 100644 index 0000000000..dff2e66d0f --- /dev/null +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -0,0 +1,211 @@ +# Copyright (c) 2023, Frappe Technologies and contributors +# For license information, please see license.txt + +import json +import re +from collections import Counter + +import sqlparse + +import frappe +from frappe import _ +from frappe.database.database import is_query_type +from frappe.model.document import Document + +RECORDER_INTERCEPT_FLAG = "recorder-intercept" +RECORDER_REQUEST_SPARSE_HASH = "recorder-requests-sparse" +RECORDER_REQUEST_HASH = "recorder-requests" +TRACEBACK_PATH_PATTERN = re.compile(".*/apps/") + + +class RecorderRequest(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.core.doctype.recorder_query.recorder_query import RecorderQuery + from frappe.types import DF + + cmd: DF.Data | None + duration: DF.Float + form_dict: DF.Code | None + method: DF.Data | None + number_of_queries: DF.Int + path: DF.Data | None + request_headers: DF.Code | None + sql_queries: DF.Table[RecorderQuery] + time: DF.Datetime | None + time_in_queries: DF.Float + # end: auto-generated types + + def db_insert(self, *args, **kwargs): + pass + + def load_from_db(self): + request_data = get(self.name) + super(Document, self).__init__(serialize_request(request_data)) + + def db_update(self): + pass + + @staticmethod + def get_list(args): + requests = [serialize_request(request) for request in get()] + return requests + + @staticmethod + def get_count(args): + pass + + @staticmethod + def get_stats(args): + pass + + @staticmethod + def delete(args): + pass + + +def administrator_only(function): + def wrapper(*args, **kwargs): + if frappe.session.user != "Administrator": + frappe.throw(_("Only Administrator is allowed to use Recorder")) + return function(*args, **kwargs) + + return wrapper + + +def do_not_record(function): + def wrapper(*args, **kwargs): + if hasattr(frappe.local, "_recorder"): + del frappe.local._recorder + frappe.db.sql = frappe.db._sql + return function(*args, **kwargs) + + return wrapper + + +@administrator_only +def get(uuid=None, *args, **kwargs): + if uuid: + result = frappe.cache.hget(RECORDER_REQUEST_HASH, uuid) + else: + result = list(frappe.cache.hgetall(RECORDER_REQUEST_SPARSE_HASH).values()) + return result + + +def serialize_request(request): + return frappe._dict( + name=request.get("uuid"), + path=request.get("path"), + method=request.get("method"), + cmd=request.get("cmd"), + number_of_queries=request.get("queries"), + time_in_queries=request.get("time_queries"), + time=request.get("time"), + duration=request.get("duration"), + request_headers=json.dumps(request.get("headers"), indent=4), + form_dict=json.dumps(request.get("form_dict"), indent=2), + sql_queries=request.get("calls"), + ) + + +@frappe.whitelist() +@do_not_record +@administrator_only +def start(*args, **kwargs): + frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1, expires_in_sec=60 * 60) + + +@frappe.whitelist() +@do_not_record +@administrator_only +def stop(*args, **kwargs): + frappe.cache.delete_value(RECORDER_INTERCEPT_FLAG) + frappe.enqueue(post_process) + + +@frappe.whitelist() +@do_not_record +@administrator_only +def delete_requests(*args, **kwargs): + frappe.cache.delete_value(RECORDER_REQUEST_SPARSE_HASH) + frappe.cache.delete_value(RECORDER_REQUEST_HASH) + + +@frappe.whitelist() +@do_not_record +@administrator_only +def get_status(*args, **kwargs): + return bool(frappe.cache.get_value(RECORDER_INTERCEPT_FLAG)) + + +def post_process(): + """post process all recorded values. + + Any processing that can be done later should be done here to avoid overhead while + profiling. As of now following values are post-processed: + - `EXPLAIN` output of queries. + - SQLParse reformatting of queries + - Mark duplicates + """ + frappe.db.rollback() + frappe.db.begin(read_only=True) # Explicitly start read only transaction + + result = list(frappe.cache.hgetall(RECORDER_REQUEST_HASH).values()) + + for request in result: + for call in request["calls"]: + formatted_query = sqlparse.format(call["query"].strip(), keyword_case="upper", reindent=True) + call["query"] = formatted_query + + # Collect EXPLAIN for executed query + if is_query_type(formatted_query, ("select", "update", "delete")): + # Only SELECT/UPDATE/DELETE queries can be "EXPLAIN"ed + try: + call["explain_result"] = frappe.db.sql(f"EXPLAIN {formatted_query}", as_dict=True) + except Exception: + pass + mark_duplicates(request) + frappe.cache.hset(RECORDER_REQUEST_HASH, request["uuid"], request) + + +def mark_duplicates(request): + exact_duplicates = Counter([call["query"] for call in request["calls"]]) + + for sql_call in request["calls"]: + sql_call["normalized_query"] = normalize_query(sql_call["query"]) + + normalized_duplicates = Counter([call["normalized_query"] for call in request["calls"]]) + + for index, call in enumerate(request["calls"]): + call["index"] = index + call["exact_copies"] = exact_duplicates[call["query"]] + call["normalized_copies"] = normalized_duplicates[call["normalized_query"]] + + +def normalize_query(query: str) -> str: + """Attempt to normalize query by removing variables. + This gives a different view of similar duplicate queries. + + Example: + These two are distinct queries: + `select * from user where name = 'x'` + `select * from user where name = 'z'` + + But their "normalized" form would be same: + `select * from user where name = ?` + """ + + try: + q = sqlparse.parse(query)[0] + for token in q.flatten(): + if "Token.Literal" in str(token.ttype): + token.value = "?" + return str(q) + except Exception as e: + print("Failed to normalize query ", e) + + return query diff --git a/frappe/core/doctype/recorder_request/recorder_request_list.js b/frappe/core/doctype/recorder_request/recorder_request_list.js new file mode 100644 index 0000000000..3007c3c5cb --- /dev/null +++ b/frappe/core/doctype/recorder_request/recorder_request_list.js @@ -0,0 +1,47 @@ +frappe.listview_settings["Recorder Request"] = { + hide_name_column: true, + + onload(listview) { + listview.page.sidebar.remove(); + if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return; + + frappe + .xcall("frappe.core.doctype.recorder_request.recorder_request.get_status") + .then((status) => { + if (status) { + listview.page.set_indicator(__("Active"), "green"); + } else { + listview.page.set_indicator(__("Inactive"), "red"); + } + }); + + listview.page.add_button(__("Start"), () => { + frappe.call({ + method: "frappe.core.doctype.recorder_request.recorder_request.start", + callback: function () { + listview.page.set_indicator(__("Active"), "green"); + listview.refresh(); + }, + }); + }); + + listview.page.add_button(__("Stop"), () => { + frappe.call({ + method: "frappe.core.doctype.recorder_request.recorder_request.stop", + callback: function () { + listview.page.set_indicator(__("Inactive"), "red"); + listview.refresh(); + }, + }); + }); + + listview.page.add_button(__("Clear"), () => { + frappe.call({ + method: "frappe.core.doctype.recorder_request.recorder_request.delete_requests", + callback: function () { + listview.refresh(); + }, + }); + }); + }, +}; diff --git a/frappe/core/doctype/recorder_request/test_recorder_request.py b/frappe/core/doctype/recorder_request/test_recorder_request.py new file mode 100644 index 0000000000..385913faae --- /dev/null +++ b/frappe/core/doctype/recorder_request/test_recorder_request.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestRecorderRequest(FrappeTestCase): + pass From 75d7fcb181ef4ab8e8cc6246315a6cd3e372fe44 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Thu, 3 Aug 2023 12:05:39 +0530 Subject: [PATCH 03/24] feat: add child table for sql queries --- .../core/doctype/recorder_query/__init__.py | 0 .../doctype/recorder_query/recorder_query.js | 8 +++ .../recorder_query/recorder_query.json | 66 +++++++++++++++++++ .../doctype/recorder_query/recorder_query.py | 50 ++++++++++++++ .../recorder_query/test_recorder_query.py | 9 +++ 5 files changed, 133 insertions(+) create mode 100644 frappe/core/doctype/recorder_query/__init__.py create mode 100644 frappe/core/doctype/recorder_query/recorder_query.js create mode 100644 frappe/core/doctype/recorder_query/recorder_query.json create mode 100644 frappe/core/doctype/recorder_query/recorder_query.py create mode 100644 frappe/core/doctype/recorder_query/test_recorder_query.py diff --git a/frappe/core/doctype/recorder_query/__init__.py b/frappe/core/doctype/recorder_query/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/recorder_query/recorder_query.js b/frappe/core/doctype/recorder_query/recorder_query.js new file mode 100644 index 0000000000..6cfeb48944 --- /dev/null +++ b/frappe/core/doctype/recorder_query/recorder_query.js @@ -0,0 +1,8 @@ +// Copyright (c) 2023, Frappe Technologies and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Recorder Query", { +// refresh(frm) { + +// }, +// }); diff --git a/frappe/core/doctype/recorder_query/recorder_query.json b/frappe/core/doctype/recorder_query/recorder_query.json new file mode 100644 index 0000000000..921d6028ef --- /dev/null +++ b/frappe/core/doctype/recorder_query/recorder_query.json @@ -0,0 +1,66 @@ +{ + "actions": [], + "creation": "2023-08-01 17:04:12.173774", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "query", + "duration", + "exact_copies", + "column_break_qmju", + "normalized_query", + "normalized_copies" + ], + "fields": [ + { + "fieldname": "query", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Query", + "length": 2, + "reqd": 1 + }, + { + "fieldname": "normalized_query", + "fieldtype": "Data", + "label": "Normalized Query" + }, + { + "fieldname": "duration", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Duration", + "reqd": 1 + }, + { + "fieldname": "exact_copies", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Exact Copies", + "reqd": 1 + }, + { + "fieldname": "normalized_copies", + "fieldtype": "Int", + "label": "Normalized Copies" + }, + { + "fieldname": "column_break_qmju", + "fieldtype": "Column Break" + } + ], + "index_web_pages_for_search": 1, + "is_virtual": 1, + "istable": 1, + "links": [], + "modified": "2023-08-02 17:23:43.670793", + "modified_by": "Administrator", + "module": "Core", + "name": "Recorder Query", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/recorder_query/recorder_query.py b/frappe/core/doctype/recorder_query/recorder_query.py new file mode 100644 index 0000000000..6471353ea7 --- /dev/null +++ b/frappe/core/doctype/recorder_query/recorder_query.py @@ -0,0 +1,50 @@ +# Copyright (c) 2023, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class RecorderQuery(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + duration: DF.Float + exact_copies: DF.Int + normalized_copies: DF.Int + normalized_query: DF.Data | None + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + query: DF.Data + # end: auto-generated types + pass + + def db_insert(self, *args, **kwargs): + pass + + def load_from_db(self): + pass + + def db_update(self): + pass + + @staticmethod + def get_list(args): + pass + + @staticmethod + def get_count(args): + pass + + @staticmethod + def get_stats(args): + pass + + def delete(self): + pass diff --git a/frappe/core/doctype/recorder_query/test_recorder_query.py b/frappe/core/doctype/recorder_query/test_recorder_query.py new file mode 100644 index 0000000000..a21fdcef08 --- /dev/null +++ b/frappe/core/doctype/recorder_query/test_recorder_query.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestRecorderQuery(FrappeTestCase): + pass From 379b90550d9116b26b2a42b3ca54fb901f0e273a Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Sat, 5 Aug 2023 17:02:36 +0530 Subject: [PATCH 04/24] fix: remove redundant fns from recorder request --- .../recorder_request/recorder_request.py | 157 ++---------------- 1 file changed, 12 insertions(+), 145 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py index dff2e66d0f..6c96bb6faf 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -3,19 +3,10 @@ import json import re -from collections import Counter - -import sqlparse import frappe -from frappe import _ -from frappe.database.database import is_query_type from frappe.model.document import Document - -RECORDER_INTERCEPT_FLAG = "recorder-intercept" -RECORDER_REQUEST_SPARSE_HASH = "recorder-requests-sparse" -RECORDER_REQUEST_HASH = "recorder-requests" -TRACEBACK_PATH_PATTERN = re.compile(".*/apps/") +from frappe.recorder import get class RecorderRequest(Document): @@ -45,7 +36,8 @@ class RecorderRequest(Document): def load_from_db(self): request_data = get(self.name) - super(Document, self).__init__(serialize_request(request_data)) + request = serialize_request(request_data) + super(Document, self).__init__(request) def db_update(self): pass @@ -68,144 +60,19 @@ class RecorderRequest(Document): pass -def administrator_only(function): - def wrapper(*args, **kwargs): - if frappe.session.user != "Administrator": - frappe.throw(_("Only Administrator is allowed to use Recorder")) - return function(*args, **kwargs) - - return wrapper - - -def do_not_record(function): - def wrapper(*args, **kwargs): - if hasattr(frappe.local, "_recorder"): - del frappe.local._recorder - frappe.db.sql = frappe.db._sql - return function(*args, **kwargs) - - return wrapper - - -@administrator_only -def get(uuid=None, *args, **kwargs): - if uuid: - result = frappe.cache.hget(RECORDER_REQUEST_HASH, uuid) - else: - result = list(frappe.cache.hgetall(RECORDER_REQUEST_SPARSE_HASH).values()) - return result - - def serialize_request(request): - return frappe._dict( + request = frappe._dict(request) + if request.get("calls"): + for i in request.calls: + i["stack"] = frappe.as_json(i["stack"]) + i["explain_result"] = frappe.as_json(i["explain_result"]) + request.update( name=request.get("uuid"), - path=request.get("path"), - method=request.get("method"), - cmd=request.get("cmd"), number_of_queries=request.get("queries"), time_in_queries=request.get("time_queries"), - time=request.get("time"), - duration=request.get("duration"), - request_headers=json.dumps(request.get("headers"), indent=4), - form_dict=json.dumps(request.get("form_dict"), indent=2), + request_headers=frappe.as_json(request.get("headers"), indent=4), + form_dict=frappe.as_json(request.get("form_dict"), indent=4), sql_queries=request.get("calls"), ) - -@frappe.whitelist() -@do_not_record -@administrator_only -def start(*args, **kwargs): - frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1, expires_in_sec=60 * 60) - - -@frappe.whitelist() -@do_not_record -@administrator_only -def stop(*args, **kwargs): - frappe.cache.delete_value(RECORDER_INTERCEPT_FLAG) - frappe.enqueue(post_process) - - -@frappe.whitelist() -@do_not_record -@administrator_only -def delete_requests(*args, **kwargs): - frappe.cache.delete_value(RECORDER_REQUEST_SPARSE_HASH) - frappe.cache.delete_value(RECORDER_REQUEST_HASH) - - -@frappe.whitelist() -@do_not_record -@administrator_only -def get_status(*args, **kwargs): - return bool(frappe.cache.get_value(RECORDER_INTERCEPT_FLAG)) - - -def post_process(): - """post process all recorded values. - - Any processing that can be done later should be done here to avoid overhead while - profiling. As of now following values are post-processed: - - `EXPLAIN` output of queries. - - SQLParse reformatting of queries - - Mark duplicates - """ - frappe.db.rollback() - frappe.db.begin(read_only=True) # Explicitly start read only transaction - - result = list(frappe.cache.hgetall(RECORDER_REQUEST_HASH).values()) - - for request in result: - for call in request["calls"]: - formatted_query = sqlparse.format(call["query"].strip(), keyword_case="upper", reindent=True) - call["query"] = formatted_query - - # Collect EXPLAIN for executed query - if is_query_type(formatted_query, ("select", "update", "delete")): - # Only SELECT/UPDATE/DELETE queries can be "EXPLAIN"ed - try: - call["explain_result"] = frappe.db.sql(f"EXPLAIN {formatted_query}", as_dict=True) - except Exception: - pass - mark_duplicates(request) - frappe.cache.hset(RECORDER_REQUEST_HASH, request["uuid"], request) - - -def mark_duplicates(request): - exact_duplicates = Counter([call["query"] for call in request["calls"]]) - - for sql_call in request["calls"]: - sql_call["normalized_query"] = normalize_query(sql_call["query"]) - - normalized_duplicates = Counter([call["normalized_query"] for call in request["calls"]]) - - for index, call in enumerate(request["calls"]): - call["index"] = index - call["exact_copies"] = exact_duplicates[call["query"]] - call["normalized_copies"] = normalized_duplicates[call["normalized_query"]] - - -def normalize_query(query: str) -> str: - """Attempt to normalize query by removing variables. - This gives a different view of similar duplicate queries. - - Example: - These two are distinct queries: - `select * from user where name = 'x'` - `select * from user where name = 'z'` - - But their "normalized" form would be same: - `select * from user where name = ?` - """ - - try: - q = sqlparse.parse(query)[0] - for token in q.flatten(): - if "Token.Literal" in str(token.ttype): - token.value = "?" - return str(q) - except Exception as e: - print("Failed to normalize query ", e) - - return query + return request From 6fc9cbcdcd445c65a7df76f88b07fabfbdbce9c6 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Sat, 5 Aug 2023 17:06:22 +0530 Subject: [PATCH 05/24] feat: add html fields for stack trace and sql explain --- .../recorder_query/recorder_query.json | 40 ++++++++++++++++++- .../doctype/recorder_query/recorder_query.py | 2 + .../recorder_request/recorder_request.json | 14 ++----- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/recorder_query/recorder_query.json b/frappe/core/doctype/recorder_query/recorder_query.json index 921d6028ef..7ef70e0ef3 100644 --- a/frappe/core/doctype/recorder_query/recorder_query.json +++ b/frappe/core/doctype/recorder_query/recorder_query.json @@ -10,7 +10,13 @@ "exact_copies", "column_break_qmju", "normalized_query", - "normalized_copies" + "normalized_copies", + "section_break_dygy", + "stack_html", + "stack", + "section_break_kvkb", + "sql_explain_html", + "explain_result" ], "fields": [ { @@ -48,13 +54,43 @@ { "fieldname": "column_break_qmju", "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_dygy", + "fieldtype": "Section Break" + }, + { + "fieldname": "stack", + "fieldtype": "Text", + "hidden": 1, + "print_hide": 1 + }, + { + "fieldname": "stack_html", + "fieldtype": "HTML", + "label": "Stack Trace" + }, + { + "fieldname": "section_break_kvkb", + "fieldtype": "Section Break" + }, + { + "fieldname": "explain_result", + "fieldtype": "Text", + "hidden": 1, + "print_hide": 1 + }, + { + "fieldname": "sql_explain_html", + "fieldtype": "HTML", + "label": "SQL Explain" } ], "index_web_pages_for_search": 1, "is_virtual": 1, "istable": 1, "links": [], - "modified": "2023-08-02 17:23:43.670793", + "modified": "2023-08-05 14:25:34.794204", "modified_by": "Administrator", "module": "Core", "name": "Recorder Query", diff --git a/frappe/core/doctype/recorder_query/recorder_query.py b/frappe/core/doctype/recorder_query/recorder_query.py index 6471353ea7..da07499f7b 100644 --- a/frappe/core/doctype/recorder_query/recorder_query.py +++ b/frappe/core/doctype/recorder_query/recorder_query.py @@ -16,12 +16,14 @@ class RecorderQuery(Document): duration: DF.Float exact_copies: DF.Int + explain_result: DF.Text | None normalized_copies: DF.Int normalized_query: DF.Data | None parent: DF.Data parentfield: DF.Data parenttype: DF.Data query: DF.Data + stack: DF.Text | None # end: auto-generated types pass diff --git a/frappe/core/doctype/recorder_request/recorder_request.json b/frappe/core/doctype/recorder_request/recorder_request.json index 9d46689bda..699bd7d1a5 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.json +++ b/frappe/core/doctype/recorder_request/recorder_request.json @@ -18,7 +18,6 @@ "request_headers", "section_break_sgro", "form_dict", - "section_break_4umr", "sql_queries" ], "fields": [ @@ -79,20 +78,15 @@ "label": "Form Dict" }, { - "fieldname": "section_break_4umr", - "fieldtype": "Section Break" + "fieldname": "method", + "fieldtype": "Data", + "label": "Method" }, { "fieldname": "sql_queries", "fieldtype": "Table", "label": "SQL Queries", "options": "Recorder Query" - }, - { - "fieldname": "method", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Method" } ], "hide_toolbar": 1, @@ -100,7 +94,7 @@ "index_web_pages_for_search": 1, "is_virtual": 1, "links": [], - "modified": "2023-08-02 20:36:12.191767", + "modified": "2023-08-05 14:45:04.358260", "modified_by": "Administrator", "module": "Core", "name": "Recorder Request", From 81b5b72f00d3735453249b871cdb70e5656ba5ef Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Sat, 5 Aug 2023 17:36:16 +0530 Subject: [PATCH 06/24] feat: add js for rendering html fields --- .../recorder_request/recorder_request.js | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request.js b/frappe/core/doctype/recorder_request/recorder_request.js index 59d813e2d7..a56d636b5a 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.js +++ b/frappe/core/doctype/recorder_request/recorder_request.js @@ -1,6 +1,47 @@ // Copyright (c) 2023, Frappe Technologies and contributors // For license information, please see license.txt -// frappe.ui.form.on("Recorder Request", { +frappe.ui.form.on("Recorder Query", "form_render", function (frm, cdt, cdn) { + let d = locals[cdt][cdn]; + let stack = JSON.parse(d.stack); + render_html_field(stack, "stack_html", "Stack Trace"); -// }); + let explain_result = JSON.parse(d.explain_result); + render_html_field(explain_result, "sql_explain_html", "SQL Explain"); + + function render_html_field(parsed_json, fieldname, label) { + let html = + "
"; + if (parsed_json.length == 0) { + html += ""; + } else { + html = create_html_table(parsed_json, html); + } + + let field_wrapper = + frm.fields_dict[d.parentfield].grid.grid_rows_by_docname[cdn].grid_form.fields_dict[ + fieldname + ].wrapper; + $(html).appendTo(field_wrapper); + } + + function create_html_table(table_content, html) { + html += + ""; + return html; + } +}); From 75861fa321b11688458d009024a07e516cbb1526 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Sat, 5 Aug 2023 17:40:48 +0530 Subject: [PATCH 07/24] fix: indicators and primary btn in listview --- .../recorder_request/recorder_request_list.js | 110 +++++++++++++----- 1 file changed, 79 insertions(+), 31 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request_list.js b/frappe/core/doctype/recorder_request/recorder_request_list.js index 3007c3c5cb..665338cacd 100644 --- a/frappe/core/doctype/recorder_request/recorder_request_list.js +++ b/frappe/core/doctype/recorder_request/recorder_request_list.js @@ -5,43 +5,91 @@ frappe.listview_settings["Recorder Request"] = { listview.page.sidebar.remove(); if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return; - frappe - .xcall("frappe.core.doctype.recorder_request.recorder_request.get_status") - .then((status) => { - if (status) { - listview.page.set_indicator(__("Active"), "green"); - } else { - listview.page.set_indicator(__("Inactive"), "red"); - } - }); - - listview.page.add_button(__("Start"), () => { - frappe.call({ - method: "frappe.core.doctype.recorder_request.recorder_request.start", - callback: function () { - listview.page.set_indicator(__("Active"), "green"); - listview.refresh(); - }, - }); - }); - - listview.page.add_button(__("Stop"), () => { - frappe.call({ - method: "frappe.core.doctype.recorder_request.recorder_request.stop", - callback: function () { - listview.page.set_indicator(__("Inactive"), "red"); - listview.refresh(); - }, - }); - }); - listview.page.add_button(__("Clear"), () => { frappe.call({ - method: "frappe.core.doctype.recorder_request.recorder_request.delete_requests", + method: "frappe.recorder.delete", callback: function () { listview.refresh(); }, }); }); + + listview.page.add_menu_item(__("Import"), () => { + new frappe.ui.FileUploader({ + folder: this.current_folder, + on_success: (file) => { + if (cur_list.data.length > 0) { + // don't replace existing capture + return; + } + frappe.call({ + method: "frappe.recorder.import_data", + args: { + file: file.file_url, + }, + callback: function () { + listview.refresh(); + }, + }); + }, + }); + }); + + listview.page.add_menu_item(__("Export"), () => { + frappe.call({ + method: "frappe.recorder.export_data", + callback: function (r) { + const data = r.message; + const filename = `${data[0]["uuid"]}..${data[data.length - 1]["uuid"]}.json`; + + const el = document.createElement("a"); + el.setAttribute( + "href", + "data:application/json," + encodeURIComponent(JSON.stringify(data)) + ); + el.setAttribute("download", filename); + el.click(); + }, + }); + }); + }, + + refresh(listview) { + this.update_primary_action(listview); + this.update_indicators(listview); + }, + + update_primary_action(listview) { + frappe.xcall("frappe.recorder.status").then((status) => { + if (status) { + listview.page.set_primary_action(__("Stop"), () => { + frappe.call({ + method: "frappe.recorder.stop", + callback: function () { + listview.refresh(); + }, + }); + }); + } else { + listview.page.set_primary_action(__("Start"), () => { + frappe.call({ + method: "frappe.recorder.start", + callback: function () { + listview.refresh(); + }, + }); + }); + } + }); + }, + + update_indicators(listview) { + frappe.xcall("frappe.recorder.status").then((status) => { + if (status) { + listview.page.set_indicator(__("Active"), "green"); + } else { + listview.page.set_indicator(__("Inactive"), "red"); + } + }); }, }; From 30f17f417b0508bee3ce6d39d95bcc9af11bbcd9 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Sat, 5 Aug 2023 17:44:00 +0530 Subject: [PATCH 08/24] fix: handle file uploader obj while importing --- frappe/recorder.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frappe/recorder.py b/frappe/recorder.py index 96ab502fec..850129a34a 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -274,3 +274,14 @@ def record_queries(func: Callable): return ret return wrapped + + +@frappe.whitelist() +@do_not_record +@administrator_only +def import_data(**args): + file_doc = frappe.get_doc("File", {"file_url": args.get("file")}) + file_content = json.loads(file_doc.get_content()) + for request in file_content: + frappe.cache.hset(RECORDER_REQUEST_SPARSE_HASH, request["uuid"], request) + frappe.cache.hset(RECORDER_REQUEST_HASH, request["uuid"], request) From b33ac8e74c77af72cb5dc3587e4200387959cba1 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Mon, 7 Aug 2023 18:25:20 +0530 Subject: [PATCH 09/24] fix: show idx for stack trace --- .../core/doctype/recorder_query/recorder_query.json | 11 +++++++++-- frappe/core/doctype/recorder_query/recorder_query.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/recorder_query/recorder_query.json b/frappe/core/doctype/recorder_query/recorder_query.json index 7ef70e0ef3..bb4e136178 100644 --- a/frappe/core/doctype/recorder_query/recorder_query.json +++ b/frappe/core/doctype/recorder_query/recorder_query.json @@ -5,10 +5,11 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ + "index", "query", "duration", - "exact_copies", "column_break_qmju", + "exact_copies", "normalized_query", "normalized_copies", "section_break_dygy", @@ -84,13 +85,19 @@ "fieldname": "sql_explain_html", "fieldtype": "HTML", "label": "SQL Explain" + }, + { + "fieldname": "index", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Index" } ], "index_web_pages_for_search": 1, "is_virtual": 1, "istable": 1, "links": [], - "modified": "2023-08-05 14:25:34.794204", + "modified": "2023-08-07 13:12:23.496002", "modified_by": "Administrator", "module": "Core", "name": "Recorder Query", diff --git a/frappe/core/doctype/recorder_query/recorder_query.py b/frappe/core/doctype/recorder_query/recorder_query.py index da07499f7b..185c927dbe 100644 --- a/frappe/core/doctype/recorder_query/recorder_query.py +++ b/frappe/core/doctype/recorder_query/recorder_query.py @@ -17,6 +17,7 @@ class RecorderQuery(Document): duration: DF.Float exact_copies: DF.Int explain_result: DF.Text | None + index: DF.Int normalized_copies: DF.Int normalized_query: DF.Data | None parent: DF.Data From 03637066b01db6a2262e514381ea094db02c67ed Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Mon, 7 Aug 2023 18:26:48 +0530 Subject: [PATCH 10/24] fix: filter and sort list view --- .../recorder_request/recorder_request.json | 17 ++++++++--- .../recorder_request/recorder_request.py | 30 ++++++++++++++++--- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request.json b/frappe/core/doctype/recorder_request/recorder_request.json index 699bd7d1a5..0ebc2b1054 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.json +++ b/frappe/core/doctype/recorder_request/recorder_request.json @@ -18,6 +18,7 @@ "request_headers", "section_break_sgro", "form_dict", + "section_break_9jhm", "sql_queries" ], "fields": [ @@ -25,11 +26,13 @@ "fieldname": "path", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Path" }, { "fieldname": "cmd", "fieldtype": "Data", + "in_standard_filter": 1, "label": "CMD" }, { @@ -79,14 +82,20 @@ }, { "fieldname": "method", - "fieldtype": "Data", - "label": "Method" + "fieldtype": "Select", + "in_standard_filter": 1, + "label": "Method", + "options": "GET\nPOST" }, { "fieldname": "sql_queries", "fieldtype": "Table", "label": "SQL Queries", "options": "Recorder Query" + }, + { + "fieldname": "section_break_9jhm", + "fieldtype": "Section Break" } ], "hide_toolbar": 1, @@ -94,7 +103,7 @@ "index_web_pages_for_search": 1, "is_virtual": 1, "links": [], - "modified": "2023-08-05 14:45:04.358260", + "modified": "2023-08-07 14:47:10.047280", "modified_by": "Administrator", "module": "Core", "name": "Recorder Request", @@ -110,7 +119,7 @@ "share": 1 } ], - "sort_field": "modified", + "sort_field": "duration", "sort_order": "DESC", "states": [], "title_field": "path" diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py index 6c96bb6faf..9e84416fe6 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -7,6 +7,7 @@ import re import frappe from frappe.model.document import Document from frappe.recorder import get +from frappe.utils import cint, compare, make_filter_dict class RecorderRequest(Document): @@ -22,7 +23,7 @@ class RecorderRequest(Document): cmd: DF.Data | None duration: DF.Float form_dict: DF.Code | None - method: DF.Data | None + method: DF.Literal["GET", "POST"] number_of_queries: DF.Int path: DF.Data | None request_headers: DF.Code | None @@ -44,12 +45,18 @@ class RecorderRequest(Document): @staticmethod def get_list(args): - requests = [serialize_request(request) for request in get()] - return requests + start = cint(args.get("start")) or 0 + page_length = cint(args.get("page_length")) or 20 + requests = RecorderRequest.get_filtered_requests(args)[start : start + page_length] + if args.get("order_by"): + sort_key, sort_order = args.get("order_by").split(".")[1].split(" ") + sort_key = sort_key.replace("`", "") + return sorted(requests, key=lambda r: r[sort_key], reverse=bool(sort_order == "desc")) + return sorted(requests, key=lambda r: r.duration, reverse=1) @staticmethod def get_count(args): - pass + return len(RecorderRequest.get_filtered_requests(args)) @staticmethod def get_stats(args): @@ -59,6 +66,21 @@ class RecorderRequest(Document): def delete(args): pass + @staticmethod + def get_filtered_requests(args): + filters = make_filter_dict(args.get("filters")) + requests = [serialize_request(request) for request in get()] + filtered_requests = [] + for request in requests: + filter_flag = 1 + for field in filters: + operator = "in" if filters[field][0] == "like" else filters[field][0] + if not compare(request[field], operator, filters[field][1]): + filter_flag = 0 + if filter_flag: + filtered_requests.append(request) + return filtered_requests + def serialize_request(request): request = frappe._dict(request) From e484a578d063ddd06551c9e2440016e3e4a1d885 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Tue, 8 Aug 2023 16:21:01 +0530 Subject: [PATCH 11/24] test: allow parent and child docs of same type --- frappe/core/doctype/doctype/doctype.py | 7 ++++--- frappe/core/doctype/doctype/test_doctype.py | 16 +++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 148e243ee3..e0209044bf 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -1535,10 +1535,11 @@ def validate_fields(meta): ) if not (meta.is_virtual == child_doctype_meta.is_virtual): + error_msg = " should be virtual." if meta.is_virtual else " cannot be virtual." frappe.throw( - _( - "Option {0} for field {1} - Either both or none of the parent and child doctypes should be virtual" - ).format(frappe.bold(doctype), frappe.bold(docfield.fieldname)), + _("Child Table {0} for field {1}" + error_msg).format( + frappe.bold(doctype), frappe.bold(docfield.fieldname) + ), title=_("Invalid Option"), ) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 40c55c594e..7ee587942c 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -567,20 +567,10 @@ class TestDocType(FrappeTestCase): "options": "Test Virtual DocType as Child Table", }, ) + self.assertRaises(frappe.exceptions.ValidationError, parent_doc.insert) + parent_doc.is_virtual = 1 parent_doc.insert(ignore_permissions=True) - - # create entry for parent doctype - parent_doc_entry = frappe.get_doc( - {"doctype": "Test Parent Virtual DocType", "some_fieldname": "Test"} - ) - parent_doc_entry.insert(ignore_permissions=True) - - # update the parent doc (should not abort because of any DB query to a virtual child table, as there is none) - parent_doc_entry.some_fieldname = "Test update" - parent_doc_entry.save(ignore_permissions=True) - - # delete the parent doc (should not abort because of any DB query to a virtual child table, as there is none) - parent_doc_entry.delete() + self.assertFalse(frappe.db.table_exists("Test Parent Virtual DocType")) def test_default_fieldname(self): fields = [ From 1dd07d825643799dd386e5bd0aebade044cf19d5 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Tue, 8 Aug 2023 16:22:59 +0530 Subject: [PATCH 12/24] test: crud with virtual child table --- .../recorder_request/recorder_request.py | 2 +- frappe/tests/test_virtual_doctype.py | 34 +++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py index 9e84416fe6..f1e1cf56c5 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -63,7 +63,7 @@ class RecorderRequest(Document): pass @staticmethod - def delete(args): + def delete(self): pass @staticmethod diff --git a/frappe/tests/test_virtual_doctype.py b/frappe/tests/test_virtual_doctype.py index 78f7fe4d14..f21cd140cd 100644 --- a/frappe/tests/test_virtual_doctype.py +++ b/frappe/tests/test_virtual_doctype.py @@ -11,6 +11,7 @@ from frappe.model.virtual_doctype import validate_controller from frappe.tests.utils import FrappeTestCase TEST_DOCTYPE_NAME = "VirtualDoctypeTest" +TEST_CHILD_DOCTYPE_NAME = "VirtualDoctypeTestChild" class VirtualDoctypeTest(Document): @@ -87,8 +88,22 @@ class TestVirtualDoctypes(FrappeTestCase): frappe.flags.allow_doctype_export = True cls.addClassCleanup(frappe.flags.pop, "allow_doctype_export", None) - vdt = new_doctype(name=TEST_DOCTYPE_NAME, is_virtual=1, custom=0).insert() + cdt = new_doctype(name=TEST_CHILD_DOCTYPE_NAME, is_virtual=1, istable=1, custom=0).insert() + vdt = new_doctype( + name=TEST_DOCTYPE_NAME, + is_virtual=1, + custom=0, + fields=[ + { + "label": "Child Table", + "fieldname": "child_table", + "fieldtype": "Table", + "options": TEST_CHILD_DOCTYPE_NAME, + } + ], + ).insert() cls.addClassCleanup(vdt.delete, force=True) + cls.addClassCleanup(cdt.delete, force=True) patch_virtual_doc = patch( "frappe.controllers", new={frappe.local.site: {TEST_DOCTYPE_NAME: VirtualDoctypeTest}} @@ -120,16 +135,20 @@ class TestVirtualDoctypes(FrappeTestCase): docname = frappe.response.docs[0]["name"] doc = frappe.get_doc(TEST_DOCTYPE_NAME, docname) - doc.some_fieldname = "New Data" + + doc.update({"child_table": [{"name": "child-1", "some_fieldname": "child1-field-value"}]}) savedocs(doc.as_json(), "Save") - doc.reload() - self.assertEqual(doc.some_fieldname, "New Data") + self.assertEqual(doc.child_table[0].some_fieldname, "child1-field-value") def test_multiple_doc_insert_and_get_list(self): - doc1 = frappe.get_doc(doctype=TEST_DOCTYPE_NAME, some_fieldname="first").insert() - doc2 = frappe.get_doc(doctype=TEST_DOCTYPE_NAME, some_fieldname="second").insert() + doc1 = frappe.new_doc(doctype=TEST_DOCTYPE_NAME) + doc1.append("child_table", {"name": "first", "some_fieldname": "first-value"}) + doc1.insert() + doc2 = frappe.new_doc(doctype=TEST_DOCTYPE_NAME) + doc2.append("child_table", {"name": "second", "some_fieldname": "second-value"}) + doc2.insert() docs = {doc1.name, doc2.name} @@ -146,7 +165,7 @@ class TestVirtualDoctypes(FrappeTestCase): self.assertIsInstance(VirtualDoctypeTest.get_count(args), int) def test_delete_doc(self): - doc = frappe.get_doc(doctype=TEST_DOCTYPE_NAME, some_fieldname="data").insert() + doc = frappe.get_doc(doctype=TEST_DOCTYPE_NAME).insert() frappe.delete_doc(doc.doctype, doc.name) @@ -155,3 +174,4 @@ class TestVirtualDoctypes(FrappeTestCase): def test_controller_validity(self): validate_controller(TEST_DOCTYPE_NAME) + validate_controller(TEST_CHILD_DOCTYPE_NAME) From bc3946002d353a0299e74d578e19ec7fb57d9f3a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 10:28:02 +0530 Subject: [PATCH 13/24] fix(DX): enable realtime refresh on recorder listview --- .../recorder_request/recorder_request_list.js | 32 +++++++++++++------ frappe/core/doctype/rq_job/rq_job_list.js | 4 --- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request_list.js b/frappe/core/doctype/recorder_request/recorder_request_list.js index 665338cacd..bd2b93fa3b 100644 --- a/frappe/core/doctype/recorder_request/recorder_request_list.js +++ b/frappe/core/doctype/recorder_request/recorder_request_list.js @@ -52,16 +52,31 @@ frappe.listview_settings["Recorder Request"] = { }, }); }); + + setInterval(() => { + if (listview.list_view_settings.disable_auto_refresh) { + return; + } + if (!listview.recorder_enabled) return; + + const route = frappe.get_route() || []; + if (route[0] != "List" || "Recorder Request" != route[1]) { + return; + } + + listview.refresh(); + }, 5000); }, refresh(listview) { - this.update_primary_action(listview); + this.setup_recorder_controls(listview); this.update_indicators(listview); }, - update_primary_action(listview) { + setup_recorder_controls(listview) { frappe.xcall("frappe.recorder.status").then((status) => { if (status) { + listview.recorder_enabled = true; listview.page.set_primary_action(__("Stop"), () => { frappe.call({ method: "frappe.recorder.stop", @@ -71,6 +86,7 @@ frappe.listview_settings["Recorder Request"] = { }); }); } else { + listview.recorder_enabled = false; listview.page.set_primary_action(__("Start"), () => { frappe.call({ method: "frappe.recorder.start", @@ -84,12 +100,10 @@ frappe.listview_settings["Recorder Request"] = { }, update_indicators(listview) { - frappe.xcall("frappe.recorder.status").then((status) => { - if (status) { - listview.page.set_indicator(__("Active"), "green"); - } else { - listview.page.set_indicator(__("Inactive"), "red"); - } - }); + if (listview.recorder_enabled) { + listview.page.set_indicator(__("Active"), "green"); + } else { + listview.page.set_indicator(__("Inactive"), "red"); + } }, }; diff --git a/frappe/core/doctype/rq_job/rq_job_list.js b/frappe/core/doctype/rq_job/rq_job_list.js index 7d140d668f..bfdd23377d 100644 --- a/frappe/core/doctype/rq_job/rq_job_list.js +++ b/frappe/core/doctype/rq_job/rq_job_list.js @@ -14,10 +14,6 @@ frappe.listview_settings["RQ Job"] = { __("Actions") ); - if (listview.list_view_settings) { - listview.list_view_settings.disable_sidebar_stats = 1; - } - frappe.xcall("frappe.utils.scheduler.get_scheduler_status").then(({ status }) => { if (status === "active") { listview.page.set_indicator(__("Scheduler: Active"), "green"); From 3bda258784ab90e87b25791286c1aa0567fd3fd8 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 10:53:58 +0530 Subject: [PATCH 14/24] fix: fast primary button switching --- .../recorder_request/recorder_request_list.js | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request_list.js b/frappe/core/doctype/recorder_request/recorder_request_list.js index bd2b93fa3b..a43320511e 100644 --- a/frappe/core/doctype/recorder_request/recorder_request_list.js +++ b/frappe/core/doctype/recorder_request/recorder_request_list.js @@ -57,7 +57,7 @@ frappe.listview_settings["Recorder Request"] = { if (listview.list_view_settings.disable_auto_refresh) { return; } - if (!listview.recorder_enabled) return; + if (!listview.enabled) return; const route = frappe.get_route() || []; if (route[0] != "List" || "Recorder Request" != route[1]) { @@ -69,38 +69,35 @@ frappe.listview_settings["Recorder Request"] = { }, refresh(listview) { + this.fetch_recorder_status(listview).then(() => this.refresh_controls(listview)); + }, + + refresh_controls(listview) { this.setup_recorder_controls(listview); this.update_indicators(listview); }, + fetch_recorder_status(listview) { + return frappe.xcall("frappe.recorder.status").then((status) => { + listview.enabled = Boolean(status); + }); + }, + setup_recorder_controls(listview) { - frappe.xcall("frappe.recorder.status").then((status) => { - if (status) { - listview.recorder_enabled = true; - listview.page.set_primary_action(__("Stop"), () => { - frappe.call({ - method: "frappe.recorder.stop", - callback: function () { - listview.refresh(); - }, - }); - }); - } else { - listview.recorder_enabled = false; - listview.page.set_primary_action(__("Start"), () => { - frappe.call({ - method: "frappe.recorder.start", - callback: function () { - listview.refresh(); - }, - }); - }); - } + listview.page.set_primary_action(listview.enabled ? __("Stop") : __("start"), () => { + frappe.call({ + method: listview.enabled ? "frappe.recorder.stop" : "frappe.recorder.start", + callback: function () { + listview.refresh(); + }, + }); + listview.enabled = !listview.enabled; + this.refresh_controls(listview); }); }, update_indicators(listview) { - if (listview.recorder_enabled) { + if (listview.enabled) { listview.page.set_indicator(__("Active"), "green"); } else { listview.page.set_indicator(__("Inactive"), "red"); From f7a62bc6dd968a7bef1cedd7acaa5c86848afa15 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 11:04:30 +0530 Subject: [PATCH 15/24] fix: Add all HTTP methods --- frappe/core/doctype/recorder_request/recorder_request.json | 4 ++-- frappe/core/doctype/recorder_request/recorder_request.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/recorder_request/recorder_request.json b/frappe/core/doctype/recorder_request/recorder_request.json index 0ebc2b1054..23929d1004 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.json +++ b/frappe/core/doctype/recorder_request/recorder_request.json @@ -85,7 +85,7 @@ "fieldtype": "Select", "in_standard_filter": 1, "label": "Method", - "options": "GET\nPOST" + "options": "GET\nPOST\nPUT\nDELETE\nPATCH\nHEAD\nOPTIONS" }, { "fieldname": "sql_queries", @@ -103,7 +103,7 @@ "index_web_pages_for_search": 1, "is_virtual": 1, "links": [], - "modified": "2023-08-07 14:47:10.047280", + "modified": "2023-08-10 11:04:16.513833", "modified_by": "Administrator", "module": "Core", "name": "Recorder Request", diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py index f1e1cf56c5..c772761860 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -23,7 +23,7 @@ class RecorderRequest(Document): cmd: DF.Data | None duration: DF.Float form_dict: DF.Code | None - method: DF.Literal["GET", "POST"] + method: DF.Literal["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"] number_of_queries: DF.Int path: DF.Data | None request_headers: DF.Code | None From 037f58239f1994f013df48ee52a6ab0e4d72102c Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 11:12:09 +0530 Subject: [PATCH 16/24] refactor: code cleanup and fix filters - rename recorder.get - move unused methods below - fix "like" filter - move filters out to separate function - revert doctype change --- frappe/core/doctype/doctype/doctype.py | 2 +- .../recorder_request/recorder_request.py | 52 +++++++++++-------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index e0209044bf..d5a8cc9ed8 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -923,7 +923,7 @@ class DocType(Document): self.nsm_parent_field = parent_field_name def validate_child_table(self): - if not self.get("istable") or self.is_new(): + if not self.get("istable") or self.is_new() or self.get("is_virtual"): # if the doctype is not a child table then return # if the doctype is a new doctype and also a child table then # don't move forward as it will be handled via schema diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py index c772761860..b40c5cc7d9 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -6,7 +6,7 @@ import re import frappe from frappe.model.document import Document -from frappe.recorder import get +from frappe.recorder import get as get_recorder_data from frappe.utils import cint, compare, make_filter_dict @@ -32,17 +32,11 @@ class RecorderRequest(Document): time_in_queries: DF.Float # end: auto-generated types - def db_insert(self, *args, **kwargs): - pass - def load_from_db(self): - request_data = get(self.name) + request_data = get_recorder_data(self.name) request = serialize_request(request_data) super(Document, self).__init__(request) - def db_update(self): - pass - @staticmethod def get_list(args): start = cint(args.get("start")) or 0 @@ -58,6 +52,12 @@ class RecorderRequest(Document): def get_count(args): return len(RecorderRequest.get_filtered_requests(args)) + @staticmethod + def get_filtered_requests(args): + filters = make_filter_dict(args.get("filters")) + requests = [serialize_request(request) for request in get_recorder_data()] + return [req for req in requests if _evaluate_filters(req, filters)] + @staticmethod def get_stats(args): pass @@ -66,20 +66,11 @@ class RecorderRequest(Document): def delete(self): pass - @staticmethod - def get_filtered_requests(args): - filters = make_filter_dict(args.get("filters")) - requests = [serialize_request(request) for request in get()] - filtered_requests = [] - for request in requests: - filter_flag = 1 - for field in filters: - operator = "in" if filters[field][0] == "like" else filters[field][0] - if not compare(request[field], operator, filters[field][1]): - filter_flag = 0 - if filter_flag: - filtered_requests.append(request) - return filtered_requests + def db_insert(self, *args, **kwargs): + pass + + def db_update(self): + pass def serialize_request(request): @@ -98,3 +89,20 @@ def serialize_request(request): ) return request + + +def _evaluate_filters(row, filters) -> bool: + for field in filters: + value = row[field] + operand = filters[field][1] + operator = filters[field][0] + + if operator == "like": + operator = "in" # python equivalent. + operand = operand.strip("%") + # Swap because like is "reverse IN" + value, operand = operand, value + + if not compare(value, operator, operand): + return False + return True From 92b1b994524b664009fea0db66d2c3555188087a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 11:34:58 +0530 Subject: [PATCH 17/24] fix: include creation/modified timestamp --- frappe/core/doctype/recorder_request/recorder_request.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py index b40c5cc7d9..329224b27b 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -86,6 +86,8 @@ def serialize_request(request): request_headers=frappe.as_json(request.get("headers"), indent=4), form_dict=frappe.as_json(request.get("form_dict"), indent=4), sql_queries=request.get("calls"), + modified=request.get("time"), + creation=request.get("time"), ) return request From 56edc2a202db6704b09fb072da67b39aca4f2f0b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 11:46:52 +0530 Subject: [PATCH 18/24] chore: hide comment count --- frappe/core/doctype/recorder_request/recorder_request_list.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/core/doctype/recorder_request/recorder_request_list.js b/frappe/core/doctype/recorder_request/recorder_request_list.js index a43320511e..2118c7c0de 100644 --- a/frappe/core/doctype/recorder_request/recorder_request_list.js +++ b/frappe/core/doctype/recorder_request/recorder_request_list.js @@ -5,6 +5,10 @@ frappe.listview_settings["Recorder Request"] = { listview.page.sidebar.remove(); if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return; + if (listview.list_view_settings) { + listview.list_view_settings.disable_comment_count = true; + } + listview.page.add_button(__("Clear"), () => { frappe.call({ method: "frappe.recorder.delete", From 94ee52c11b676739269bdf25a183f1bdf68c669c Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 11:54:00 +0530 Subject: [PATCH 19/24] fix: if request is deleted then 404 --- frappe/core/doctype/recorder_request/recorder_request.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder_request/recorder_request.py index 329224b27b..4d3d480cfd 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder_request/recorder_request.py @@ -34,6 +34,8 @@ class RecorderRequest(Document): def load_from_db(self): request_data = get_recorder_data(self.name) + if not request_data: + raise frappe.DoesNotExistError request = serialize_request(request_data) super(Document, self).__init__(request) From de932b464ffb650f706238359f2100cd332830b8 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 12:01:12 +0530 Subject: [PATCH 20/24] refactor: delete old recorder and rename new one --- .../__init__.py | 0 .../recorder.js} | 8 +- .../recorder.json} | 6 +- .../recorder.py} | 9 +- .../recorder_list.js} | 4 +- .../test_recorder.py} | 2 +- frappe/core/page/recorder/__init__.py | 0 frappe/core/page/recorder/recorder.js | 28 -- frappe/core/page/recorder/recorder.json | 23 -- frappe/patches.txt | 1 + .../js/frappe/recorder/RecorderDetail.vue | 306 ----------------- .../js/frappe/recorder/RecorderRoot.vue | 20 -- .../js/frappe/recorder/RequestDetail.vue | 321 ------------------ .../js/frappe/recorder/recorder.bundle.js | 8 - frappe/public/js/frappe/recorder/router.js | 28 -- 15 files changed, 14 insertions(+), 750 deletions(-) rename frappe/core/doctype/{recorder_request => recorder}/__init__.py (100%) rename frappe/core/doctype/{recorder_request/recorder_request.js => recorder/recorder.js} (85%) rename frappe/core/doctype/{recorder_request/recorder_request.json => recorder/recorder.json} (95%) rename frappe/core/doctype/{recorder_request/recorder_request.py => recorder/recorder.py} (93%) rename frappe/core/doctype/{recorder_request/recorder_request_list.js => recorder/recorder_list.js} (95%) rename frappe/core/doctype/{recorder_request/test_recorder_request.py => recorder/test_recorder.py} (77%) delete mode 100644 frappe/core/page/recorder/__init__.py delete mode 100644 frappe/core/page/recorder/recorder.js delete mode 100644 frappe/core/page/recorder/recorder.json delete mode 100644 frappe/public/js/frappe/recorder/RecorderDetail.vue delete mode 100644 frappe/public/js/frappe/recorder/RecorderRoot.vue delete mode 100644 frappe/public/js/frappe/recorder/RequestDetail.vue delete mode 100644 frappe/public/js/frappe/recorder/recorder.bundle.js delete mode 100644 frappe/public/js/frappe/recorder/router.js diff --git a/frappe/core/doctype/recorder_request/__init__.py b/frappe/core/doctype/recorder/__init__.py similarity index 100% rename from frappe/core/doctype/recorder_request/__init__.py rename to frappe/core/doctype/recorder/__init__.py diff --git a/frappe/core/doctype/recorder_request/recorder_request.js b/frappe/core/doctype/recorder/recorder.js similarity index 85% rename from frappe/core/doctype/recorder_request/recorder_request.js rename to frappe/core/doctype/recorder/recorder.js index a56d636b5a..c286abef91 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.js +++ b/frappe/core/doctype/recorder/recorder.js @@ -2,11 +2,11 @@ // For license information, please see license.txt frappe.ui.form.on("Recorder Query", "form_render", function (frm, cdt, cdn) { - let d = locals[cdt][cdn]; - let stack = JSON.parse(d.stack); + let row = frappe.get_doc(cdt, cdn); + let stack = JSON.parse(row.stack); render_html_field(stack, "stack_html", "Stack Trace"); - let explain_result = JSON.parse(d.explain_result); + let explain_result = JSON.parse(row.explain_result); render_html_field(explain_result, "sql_explain_html", "SQL Explain"); function render_html_field(parsed_json, fieldname, label) { @@ -19,7 +19,7 @@ frappe.ui.form.on("Recorder Query", "form_render", function (frm, cdt, cdn) { } let field_wrapper = - frm.fields_dict[d.parentfield].grid.grid_rows_by_docname[cdn].grid_form.fields_dict[ + frm.fields_dict[row.parentfield].grid.grid_rows_by_docname[cdn].grid_form.fields_dict[ fieldname ].wrapper; $(html).appendTo(field_wrapper); diff --git a/frappe/core/doctype/recorder_request/recorder_request.json b/frappe/core/doctype/recorder/recorder.json similarity index 95% rename from frappe/core/doctype/recorder_request/recorder_request.json rename to frappe/core/doctype/recorder/recorder.json index 23929d1004..aa0d782811 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.json +++ b/frappe/core/doctype/recorder/recorder.json @@ -103,10 +103,10 @@ "index_web_pages_for_search": 1, "is_virtual": 1, "links": [], - "modified": "2023-08-10 11:04:16.513833", + "modified": "2023-08-10 12:01:03.456643", "modified_by": "Administrator", "module": "Core", - "name": "Recorder Request", + "name": "Recorder", "owner": "Administrator", "permissions": [ { @@ -115,7 +115,7 @@ "print": 1, "read": 1, "report": 1, - "role": "System Manager", + "role": "Administrator", "share": 1 } ], diff --git a/frappe/core/doctype/recorder_request/recorder_request.py b/frappe/core/doctype/recorder/recorder.py similarity index 93% rename from frappe/core/doctype/recorder_request/recorder_request.py rename to frappe/core/doctype/recorder/recorder.py index 4d3d480cfd..89a9c770ae 100644 --- a/frappe/core/doctype/recorder_request/recorder_request.py +++ b/frappe/core/doctype/recorder/recorder.py @@ -1,16 +1,13 @@ # Copyright (c) 2023, Frappe Technologies and contributors # For license information, please see license.txt -import json -import re - import frappe from frappe.model.document import Document from frappe.recorder import get as get_recorder_data from frappe.utils import cint, compare, make_filter_dict -class RecorderRequest(Document): +class Recorder(Document): # begin: auto-generated types # This code is auto-generated. Do not modify anything in this block. @@ -43,7 +40,7 @@ class RecorderRequest(Document): def get_list(args): start = cint(args.get("start")) or 0 page_length = cint(args.get("page_length")) or 20 - requests = RecorderRequest.get_filtered_requests(args)[start : start + page_length] + requests = Recorder.get_filtered_requests(args)[start : start + page_length] if args.get("order_by"): sort_key, sort_order = args.get("order_by").split(".")[1].split(" ") sort_key = sort_key.replace("`", "") @@ -52,7 +49,7 @@ class RecorderRequest(Document): @staticmethod def get_count(args): - return len(RecorderRequest.get_filtered_requests(args)) + return len(Recorder.get_filtered_requests(args)) @staticmethod def get_filtered_requests(args): diff --git a/frappe/core/doctype/recorder_request/recorder_request_list.js b/frappe/core/doctype/recorder/recorder_list.js similarity index 95% rename from frappe/core/doctype/recorder_request/recorder_request_list.js rename to frappe/core/doctype/recorder/recorder_list.js index 2118c7c0de..eb3f73e77b 100644 --- a/frappe/core/doctype/recorder_request/recorder_request_list.js +++ b/frappe/core/doctype/recorder/recorder_list.js @@ -1,4 +1,4 @@ -frappe.listview_settings["Recorder Request"] = { +frappe.listview_settings["Recorder"] = { hide_name_column: true, onload(listview) { @@ -64,7 +64,7 @@ frappe.listview_settings["Recorder Request"] = { if (!listview.enabled) return; const route = frappe.get_route() || []; - if (route[0] != "List" || "Recorder Request" != route[1]) { + if (route[0] != "List" || "Recorder" != route[1]) { return; } diff --git a/frappe/core/doctype/recorder_request/test_recorder_request.py b/frappe/core/doctype/recorder/test_recorder.py similarity index 77% rename from frappe/core/doctype/recorder_request/test_recorder_request.py rename to frappe/core/doctype/recorder/test_recorder.py index 385913faae..aa2bb52c98 100644 --- a/frappe/core/doctype/recorder_request/test_recorder_request.py +++ b/frappe/core/doctype/recorder/test_recorder.py @@ -5,5 +5,5 @@ from frappe.tests.utils import FrappeTestCase -class TestRecorderRequest(FrappeTestCase): +class TestRecorder(FrappeTestCase): pass diff --git a/frappe/core/page/recorder/__init__.py b/frappe/core/page/recorder/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/core/page/recorder/recorder.js b/frappe/core/page/recorder/recorder.js deleted file mode 100644 index 1f004915fe..0000000000 --- a/frappe/core/page/recorder/recorder.js +++ /dev/null @@ -1,28 +0,0 @@ -frappe.pages["recorder"].on_page_load = function (wrapper) { - frappe.ui.make_app_page({ - parent: wrapper, - title: __("Recorder"), - single_column: true, - card_layout: true, - }); - - frappe.recorder = new Recorder(wrapper); - $(wrapper).bind("show", function () { - frappe.recorder.show(); - }); - - frappe.require("recorder.bundle.js"); -}; - -class Recorder { - constructor(wrapper) { - this.wrapper = $(wrapper); - this.container = this.wrapper.find(".layout-main-section"); - this.container.append($('
')); - } - - show() { - if (!this.route || this.route.name == "RecorderDetail") return; - this.router?.replace({ name: "RecorderDetail" }); - } -} diff --git a/frappe/core/page/recorder/recorder.json b/frappe/core/page/recorder/recorder.json deleted file mode 100644 index 43dfbc0e09..0000000000 --- a/frappe/core/page/recorder/recorder.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "content": null, - "creation": "2019-02-08 08:17:45.392739", - "docstatus": 0, - "doctype": "Page", - "idx": 0, - "modified": "2019-02-08 08:23:04.416426", - "modified_by": "Administrator", - "module": "Core", - "name": "recorder", - "owner": "Administrator", - "page_name": "Recorder", - "roles": [ - { - "role": "Administrator" - } - ], - "script": null, - "standard": "Yes", - "style": null, - "system_page": 0, - "title": "Recorder" -} \ No newline at end of file diff --git a/frappe/patches.txt b/frappe/patches.txt index 054fe9b946..c16bf8d1d7 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -193,6 +193,7 @@ frappe.patches.v14_0.delete_payment_gateways frappe.patches.v15_0.remove_event_streaming frappe.patches.v15_0.copy_disable_prepared_report_to_prepared_report execute:frappe.reload_doc("desk", "doctype", "Form Tour") +execute:frappe.delete_doc('Page', 'recorder', ignore_missing=True, force=True) [post_model_sync] execute:frappe.get_doc('Role', 'Guest').save() # remove desk access diff --git a/frappe/public/js/frappe/recorder/RecorderDetail.vue b/frappe/public/js/frappe/recorder/RecorderDetail.vue deleted file mode 100644 index c201c1528e..0000000000 --- a/frappe/public/js/frappe/recorder/RecorderDetail.vue +++ /dev/null @@ -1,306 +0,0 @@ - - - - - diff --git a/frappe/public/js/frappe/recorder/RecorderRoot.vue b/frappe/public/js/frappe/recorder/RecorderRoot.vue deleted file mode 100644 index 7c802ccec1..0000000000 --- a/frappe/public/js/frappe/recorder/RecorderRoot.vue +++ /dev/null @@ -1,20 +0,0 @@ - - - diff --git a/frappe/public/js/frappe/recorder/RequestDetail.vue b/frappe/public/js/frappe/recorder/RequestDetail.vue deleted file mode 100644 index 7d6e89f7ed..0000000000 --- a/frappe/public/js/frappe/recorder/RequestDetail.vue +++ /dev/null @@ -1,321 +0,0 @@ - - - diff --git a/frappe/public/js/frappe/recorder/recorder.bundle.js b/frappe/public/js/frappe/recorder/recorder.bundle.js deleted file mode 100644 index 4f6fb59d26..0000000000 --- a/frappe/public/js/frappe/recorder/recorder.bundle.js +++ /dev/null @@ -1,8 +0,0 @@ -import { createApp } from "vue"; -import RecorderRoot from "./RecorderRoot.vue"; -import router from "./router.js"; - -let app = createApp(RecorderRoot).use(router); -SetVueGlobals(app); -app.mount(".recorder-container"); -frappe.recorder.view = app; diff --git a/frappe/public/js/frappe/recorder/router.js b/frappe/public/js/frappe/recorder/router.js deleted file mode 100644 index ebff6eba7c..0000000000 --- a/frappe/public/js/frappe/recorder/router.js +++ /dev/null @@ -1,28 +0,0 @@ -import { createWebHistory, createRouter } from "vue-router"; -import RecorderDetail from "./RecorderDetail.vue"; -import RequestDetail from "./RequestDetail.vue"; - -const routes = [ - { - path: "/detail", - name: "RecorderDetail", - component: RecorderDetail, - }, - { - path: "/request/:id", - name: "RequestDetail", - component: RequestDetail, - meta: { shouldFetch: true }, - }, - { - path: "/", - redirect: "/detail", - }, -]; - -const router = createRouter({ - history: createWebHistory("/app/recorder/"), - routes, -}); - -export default router; From 8ffd363ccf2147164b4e6b18531a41177841b102 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 12:40:23 +0530 Subject: [PATCH 21/24] fix: missing order by clause and tests --- frappe/core/doctype/recorder/recorder.py | 16 +++++++++--- frappe/core/doctype/recorder/test_recorder.py | 26 +++++++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/recorder/recorder.py b/frappe/core/doctype/recorder/recorder.py index 89a9c770ae..4aab095914 100644 --- a/frappe/core/doctype/recorder/recorder.py +++ b/frappe/core/doctype/recorder/recorder.py @@ -41,10 +41,20 @@ class Recorder(Document): start = cint(args.get("start")) or 0 page_length = cint(args.get("page_length")) or 20 requests = Recorder.get_filtered_requests(args)[start : start + page_length] - if args.get("order_by"): - sort_key, sort_order = args.get("order_by").split(".")[1].split(" ") + + if order_by_statment := args.get("order_by"): + if "." in order_by_statment: + order_by_statment = order_by_statment.split(".")[1] + + if " " in order_by_statment: + sort_key, sort_order = order_by_statment.split(" ") + else: + sort_key = order_by_statment + sort_order = "desc" + sort_key = sort_key.replace("`", "") - return sorted(requests, key=lambda r: r[sort_key], reverse=bool(sort_order == "desc")) + return sorted(requests, key=lambda r: r.get(sort_key) or 0, reverse=bool(sort_order == "desc")) + return sorted(requests, key=lambda r: r.duration, reverse=1) @staticmethod diff --git a/frappe/core/doctype/recorder/test_recorder.py b/frappe/core/doctype/recorder/test_recorder.py index aa2bb52c98..6069e9d47f 100644 --- a/frappe/core/doctype/recorder/test_recorder.py +++ b/frappe/core/doctype/recorder/test_recorder.py @@ -1,9 +1,31 @@ # Copyright (c) 2023, Frappe Technologies and Contributors # See license.txt -# import frappe +import frappe +import frappe.recorder from frappe.tests.utils import FrappeTestCase +from frappe.utils import set_request class TestRecorder(FrappeTestCase): - pass + def setUp(self): + self.start_recoder() + + def start_recoder(self): + frappe.recorder.stop() + frappe.recorder.delete() + set_request() + frappe.recorder.start() + frappe.recorder.record() + + def stop_recorder(self): + frappe.recorder.dump() + + def test_recorder_list(self): + frappe.get_all("User") # trigger one query + self.stop_recorder() + + requests = frappe.get_all("Recorder") + self.assertGreaterEqual(len(requests), 1) + request = frappe.get_doc("Recorder", requests[0].name) + self.assertGreaterEqual(len(request.sql_queries), 1) From 40c5dc54264e016ded8108be11678e28253a94a6 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 12:50:51 +0530 Subject: [PATCH 22/24] refactor: flatten file argument in function call --- frappe/recorder.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/recorder.py b/frappe/recorder.py index 850129a34a..402175aa50 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -193,6 +193,7 @@ def _unpatch(): def do_not_record(function): + @functools.wraps(function) def wrapper(*args, **kwargs): if hasattr(frappe.local, "_recorder"): del frappe.local._recorder @@ -203,6 +204,7 @@ def do_not_record(function): def administrator_only(function): + @functools.wraps(function) def wrapper(*args, **kwargs): if frappe.session.user != "Administrator": frappe.throw(_("Only Administrator is allowed to use Recorder")) @@ -279,9 +281,10 @@ def record_queries(func: Callable): @frappe.whitelist() @do_not_record @administrator_only -def import_data(**args): - file_doc = frappe.get_doc("File", {"file_url": args.get("file")}) +def import_data(file: str) -> None: + file_doc = frappe.get_doc("File", {"file_url": file}) file_content = json.loads(file_doc.get_content()) for request in file_content: frappe.cache.hset(RECORDER_REQUEST_SPARSE_HASH, request["uuid"], request) frappe.cache.hset(RECORDER_REQUEST_HASH, request["uuid"], request) + file_doc.delete() From 074bed9ad67eeb813d459dbaa908a0d2f70e991f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 10 Aug 2023 12:57:28 +0530 Subject: [PATCH 23/24] fix: delete imported recorder file after import --- frappe/recorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/recorder.py b/frappe/recorder.py index 6c88b2007f..2bc14e9f2f 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -287,4 +287,4 @@ def import_data(file: str) -> None: for request in file_content: frappe.cache.hset(RECORDER_REQUEST_SPARSE_HASH, request["uuid"], request) frappe.cache.hset(RECORDER_REQUEST_HASH, request["uuid"], request) - file_doc.delete() + file_doc.delete(delete_permanently=True) From 67111346c3b91e85b413f35449dc858410b12464 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Thu, 10 Aug 2023 20:50:36 +0530 Subject: [PATCH 24/24] test: recorder filter sort and serialization --- frappe/core/doctype/recorder/recorder_list.js | 2 +- frappe/core/doctype/recorder/test_recorder.py | 47 ++++++++++++++++++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/recorder/recorder_list.js b/frappe/core/doctype/recorder/recorder_list.js index eb3f73e77b..a0eadae260 100644 --- a/frappe/core/doctype/recorder/recorder_list.js +++ b/frappe/core/doctype/recorder/recorder_list.js @@ -88,7 +88,7 @@ frappe.listview_settings["Recorder"] = { }, setup_recorder_controls(listview) { - listview.page.set_primary_action(listview.enabled ? __("Stop") : __("start"), () => { + listview.page.set_primary_action(listview.enabled ? __("Stop") : __("Start"), () => { frappe.call({ method: listview.enabled ? "frappe.recorder.stop" : "frappe.recorder.start", callback: function () { diff --git a/frappe/core/doctype/recorder/test_recorder.py b/frappe/core/doctype/recorder/test_recorder.py index 6069e9d47f..d0dfc3827b 100644 --- a/frappe/core/doctype/recorder/test_recorder.py +++ b/frappe/core/doctype/recorder/test_recorder.py @@ -1,8 +1,12 @@ # Copyright (c) 2023, Frappe Technologies and Contributors # See license.txt +import re + import frappe import frappe.recorder +from frappe.core.doctype.recorder.recorder import serialize_request +from frappe.recorder import get as get_recorder_data from frappe.tests.utils import FrappeTestCase from frappe.utils import set_request @@ -14,7 +18,7 @@ class TestRecorder(FrappeTestCase): def start_recoder(self): frappe.recorder.stop() frappe.recorder.delete() - set_request() + set_request(path="/api/method/ping") frappe.recorder.start() frappe.recorder.record() @@ -24,8 +28,47 @@ class TestRecorder(FrappeTestCase): def test_recorder_list(self): frappe.get_all("User") # trigger one query self.stop_recorder() - requests = frappe.get_all("Recorder") self.assertGreaterEqual(len(requests), 1) request = frappe.get_doc("Recorder", requests[0].name) self.assertGreaterEqual(len(request.sql_queries), 1) + queries = [sql_query.query for sql_query in request.sql_queries] + match_flag = 0 + for query in queries: + if bool(re.match("^[select.*from `tabUser`]", query, flags=re.IGNORECASE)): + match_flag = 1 + break + self.assertEqual(match_flag, 1) + + def test_recorder_list_filters(self): + user = frappe.qb.DocType("User") + frappe.qb.from_(user).select("name").run() + self.stop_recorder() + + set_request(path="/api/method/abc") + frappe.recorder.start() + frappe.recorder.record() + frappe.get_all("User") + self.stop_recorder() + + requests = frappe.get_list( + "Recorder", filters={"path": ("like", "/api/method/ping"), "number_of_queries": 1} + ) + self.assertGreaterEqual(len(requests), 1) + requests = frappe.get_list("Recorder", filters={"path": ("like", "/api/method/test")}) + self.assertEqual(len(requests), 0) + + requests = frappe.get_list("Recorder", filters={"method": "GET"}) + self.assertGreaterEqual(len(requests), 1) + requests = frappe.get_list("Recorder", filters={"method": "POST"}) + self.assertEqual(len(requests), 0) + + requests = frappe.get_list("Recorder", order_by="path desc") + self.assertEqual(requests[0].path, "/api/method/ping") + + def test_recorder_serialization(self): + frappe.get_all("User") # trigger one query + self.stop_recorder() + requests = frappe.get_all("Recorder") + request_doc = get_recorder_data(requests[0].name) + self.assertIsInstance(serialize_request(request_doc), dict)