From f1a60d99fe6317046e1af2ad8edf40036f0e97f1 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 18 Dec 2024 11:58:29 +0530 Subject: [PATCH 1/6] feat: analytics on Prepared Reports --- .../prepared_report_analytics/__init__.py | 0 .../prepared_report_analytics.js | 13 +++++ .../prepared_report_analytics.json | 30 ++++++++++++ .../prepared_report_analytics.py | 48 +++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 frappe/core/report/prepared_report_analytics/__init__.py create mode 100644 frappe/core/report/prepared_report_analytics/prepared_report_analytics.js create mode 100644 frappe/core/report/prepared_report_analytics/prepared_report_analytics.json create mode 100644 frappe/core/report/prepared_report_analytics/prepared_report_analytics.py diff --git a/frappe/core/report/prepared_report_analytics/__init__.py b/frappe/core/report/prepared_report_analytics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.js b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.js new file mode 100644 index 0000000000..6e0af20626 --- /dev/null +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.js @@ -0,0 +1,13 @@ +// Copyright (c) 2024, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.query_reports["Prepared Report Analytics"] = { + filters: [ + // { + // "fieldname": "my_filter", + // "label": __("My Filter"), + // "fieldtype": "Data", + // "reqd": 1, + // }, + ], +}; diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.json b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.json new file mode 100644 index 0000000000..911e4fbfc5 --- /dev/null +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.json @@ -0,0 +1,30 @@ +{ + "add_total_row": 0, + "columns": [], + "creation": "2024-12-18 11:58:00.693755", + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "filters": [], + "idx": 0, + "is_standard": "Yes", + "letterhead": null, + "modified": "2024-12-18 11:58:00.693755", + "modified_by": "Administrator", + "module": "Core", + "name": "Prepared Report Analytics", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "Prepared Report", + "report_name": "Prepared Report Analytics", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + }, + { + "role": "Prepared Report User" + } + ], + "timeout": 0 +} \ No newline at end of file diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py new file mode 100644 index 0000000000..9eeaf1f9dd --- /dev/null +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe import _ + + +def execute(filters: dict | None = None): + """Return columns and data for the report. + + This is the main entry point for the report. It accepts the filters as a + dictionary and should return columns and data. It is called by the framework + every time the report is refreshed or a filter is updated. + """ + columns = get_columns() + data = get_data() + + return columns, data + + +def get_columns() -> list[dict]: + """Return columns for the report. + + One field definition per column, just like a DocType field definition. + """ + return [ + { + "label": _("Column 1"), + "fieldname": "column_1", + "fieldtype": "Data", + }, + { + "label": _("Column 2"), + "fieldname": "column_2", + "fieldtype": "Int", + }, + ] + + +def get_data() -> list[list]: + """Return data for the report. + + The report data is a list of rows, with each row being a list of cell values. + """ + return [ + ["Row 1", 1], + ["Row 2", 2], + ] From 7c4265f7fbd37e8c757d1718bffd54a8b1ed1005 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 18 Dec 2024 15:41:32 +0530 Subject: [PATCH 2/6] refactor: basic report with filters --- .../prepared_report_analytics.js | 23 ++++-- .../prepared_report_analytics.py | 78 +++++++++++++++---- 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.js b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.js index 6e0af20626..d659f77e59 100644 --- a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.js +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.js @@ -3,11 +3,22 @@ frappe.query_reports["Prepared Report Analytics"] = { filters: [ - // { - // "fieldname": "my_filter", - // "label": __("My Filter"), - // "fieldtype": "Data", - // "reqd": 1, - // }, + { + fieldname: "report", + label: __("Report"), + fieldtype: "Data", + }, + { + fieldname: "top_10", + label: __("Top 10"), + fieldtype: "Check", + default: 0, + }, + { + fieldname: "in_minutes", + label: __("In Minutes"), + fieldtype: "Check", + default: 0, + }, ], }; diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py index 9eeaf1f9dd..922fc26df5 100644 --- a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py @@ -1,8 +1,12 @@ # Copyright (c) 2024, Frappe Technologies and contributors # For license information, please see license.txt -# import frappe -from frappe import _ +from pypika import Order + +import frappe +from frappe import _, qb +from frappe.query_builder import Criterion +from frappe.utils import add_months, nowdate def execute(filters: dict | None = None): @@ -12,37 +16,81 @@ def execute(filters: dict | None = None): dictionary and should return columns and data. It is called by the framework every time the report is refreshed or a filter is updated. """ - columns = get_columns() - data = get_data() + columns = get_columns(filters) + data = get_data(filters=filters) return columns, data -def get_columns() -> list[dict]: +def get_columns(filters) -> list[dict]: """Return columns for the report. One field definition per column, just like a DocType field definition. """ return [ { - "label": _("Column 1"), - "fieldname": "column_1", - "fieldtype": "Data", + "label": _("Prepared Report"), + "fieldname": "name", + "fieldtype": "Link", + "options": "Prepared Report", + "width": 250, }, { - "label": _("Column 2"), - "fieldname": "column_2", + "label": _("Report Name"), + "fieldname": "report_name", + "fieldtype": "Data", + "width": 250, + }, + { + "label": _("Start"), + "fieldname": "creation", + "fieldtype": "DateTime", + "width": 250, + }, + { + "label": _("End"), + "fieldname": "report_end_time", + "fieldtype": "DateTime", + "width": 250, + }, + { + "label": _("Runtime in Minutes") if filters.in_minutes else _("Runtime in Seconds"), + "fieldname": "runtime", "fieldtype": "Int", + "width": 250, }, ] -def get_data() -> list[list]: +def get_data(filters) -> list[list]: """Return data for the report. The report data is a list of rows, with each row being a list of cell values. """ - return [ - ["Row 1", 1], - ["Row 2", 2], - ] + + pr = qb.DocType("Prepared Report") + + conditions = [pr.status.eq("Completed"), pr.creation.gte(add_months(nowdate(), -2))] + + if filters.report: + conditions.append(pr.report_name.like(f"%{filters.report}%")) + + res: list = ( + qb.from_(pr) + .select(pr.name, pr.report_name, pr.creation, pr.report_end_time) + .where(Criterion.all(conditions)) + .orderby(pr.creation, order=Order.desc) + .run(as_dict=True) + ) + + divisor = 1 + if filters.in_minutes: + divisor = 60 + + for x in res: + x.runtime = ((x.report_end_time - x.creation).total_seconds()) / divisor + + res.sort(key=lambda x: x.runtime, reverse=True) + if filters.top_10: + res = res[:10] + return res From cb6e561edbeafead6cc8a276adc169703f0f51a4 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 7 Feb 2025 16:37:06 +0530 Subject: [PATCH 3/6] refactor: include peak memory usage --- .../prepared_report_analytics.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py index 922fc26df5..9436b77d6a 100644 --- a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py @@ -59,6 +59,12 @@ def get_columns(filters) -> list[dict]: "fieldtype": "Int", "width": 250, }, + { + "label": _("Peak Memory Usage"), + "fieldname": "peak_memory_usage", + "fieldtype": "Int", + "width": 250, + }, ] @@ -77,7 +83,7 @@ def get_data(filters) -> list[list]: res: list = ( qb.from_(pr) - .select(pr.name, pr.report_name, pr.creation, pr.report_end_time) + .select(pr.name, pr.report_name, pr.creation, pr.report_end_time, pr.peak_memory_usage) .where(Criterion.all(conditions)) .orderby(pr.creation, order=Order.desc) .run(as_dict=True) From 4fcde7d3d41b06c750bfa78876accb93755d7a09 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 7 Feb 2025 17:25:01 +0530 Subject: [PATCH 4/6] refactor: more changes --- .../prepared_report_analytics.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py index 9436b77d6a..d181f9c332 100644 --- a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py @@ -56,13 +56,13 @@ def get_columns(filters) -> list[dict]: { "label": _("Runtime in Minutes") if filters.in_minutes else _("Runtime in Seconds"), "fieldname": "runtime", - "fieldtype": "Int", + "fieldtype": "float", "width": 250, }, { - "label": _("Peak Memory Usage"), + "label": _("Memory Usage in MB"), "fieldname": "peak_memory_usage", - "fieldtype": "Int", + "fieldtype": "float", "width": 250, }, ] @@ -81,20 +81,24 @@ def get_data(filters) -> list[list]: if filters.report: conditions.append(pr.report_name.like(f"%{filters.report}%")) - res: list = ( - qb.from_(pr) - .select(pr.name, pr.report_name, pr.creation, pr.report_end_time, pr.peak_memory_usage) - .where(Criterion.all(conditions)) - .orderby(pr.creation, order=Order.desc) - .run(as_dict=True) - ) - divisor = 1 if filters.in_minutes: divisor = 60 - for x in res: - x.runtime = ((x.report_end_time - x.creation).total_seconds()) / divisor + res: list = ( + qb.from_(pr) + .select( + pr.name, + pr.report_name, + pr.creation, + pr.report_end_time, + (pr.peak_memory_usage / 1024).as_("peak_memory_usage"), + ) + .select(((pr.report_end_time - pr.creation) / divisor).as_("runtime")) + .where(Criterion.all(conditions)) + .orderby(qb.Field("runtime"), order=Order.desc) + .run(as_dict=True) + ) res.sort(key=lambda x: x.runtime, reverse=True) if filters.top_10: From 966bf89aa53bb30b5d246823c8f6188573b44e4a Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 7 Feb 2025 17:27:58 +0530 Subject: [PATCH 5/6] refactor: restrict access to system manager --- .../prepared_report_analytics/prepared_report_analytics.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.json b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.json index 911e4fbfc5..49e0cbc262 100644 --- a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.json +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.json @@ -9,7 +9,7 @@ "idx": 0, "is_standard": "Yes", "letterhead": null, - "modified": "2024-12-18 11:58:00.693755", + "modified": "2025-02-07 17:27:53.441631", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report Analytics", @@ -21,9 +21,6 @@ "roles": [ { "role": "System Manager" - }, - { - "role": "Prepared Report User" } ], "timeout": 0 From 8de1483ef4745775e37ca76bb2c341e5b1e1ce71 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 7 Feb 2025 17:53:26 +0530 Subject: [PATCH 6/6] refactor: limit in query --- .../prepared_report_analytics.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py index d181f9c332..b7f564e2fe 100644 --- a/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py +++ b/frappe/core/report/prepared_report_analytics/prepared_report_analytics.py @@ -85,7 +85,7 @@ def get_data(filters) -> list[list]: if filters.in_minutes: divisor = 60 - res: list = ( + query = ( qb.from_(pr) .select( pr.name, @@ -97,10 +97,10 @@ def get_data(filters) -> list[list]: .select(((pr.report_end_time - pr.creation) / divisor).as_("runtime")) .where(Criterion.all(conditions)) .orderby(qb.Field("runtime"), order=Order.desc) - .run(as_dict=True) ) - - res.sort(key=lambda x: x.runtime, reverse=True) if filters.top_10: - res = res[:10] + query = query.limit(10) + + res = query.run(as_dict=True) + return res