diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index f0f2cdaae8..e1e1d5d176 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -18,6 +18,7 @@ DEFAULT_LOGTYPES_RETENTION = { "Scheduled Job Log": 90, "Route History": 90, "Submission Queue": 30, + "Prepared Report": 30, } @@ -155,7 +156,6 @@ LOG_DOCTYPES = [ "Email Queue Recipient", "Error Snapshot", "Error Log", - "Submission Queue", ] diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index f99a7d41d0..c3b21e42a9 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -22,9 +22,8 @@ "fields": [ { "fieldname": "report_name", - "fieldtype": "Link", + "fieldtype": "Data", "label": "Report Name", - "options": "Report", "read_only": 1, "reqd": 1 }, @@ -99,7 +98,7 @@ ], "in_create": 1, "links": [], - "modified": "2022-11-18 21:52:45.444209", + "modified": "2022-11-20 15:15:00.907626", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py index 2f31cd5139..cff9296558 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.py +++ b/frappe/core/doctype/prepared_report/prepared_report.py @@ -16,6 +16,16 @@ from frappe.utils.background_jobs import enqueue class PreparedReport(Document): + @staticmethod + def clear_old_logs(days=30): + prepared_reports_to_delete = frappe.get_all( + "Prepared Report", + filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]}, + ) + + for batch in frappe.utils.create_batch(prepared_reports_to_delete, 100): + enqueue(method=delete_prepared_reports, reports=batch) + def before_insert(self): self.status = "Queued" self.report_start_time = frappe.utils.now() @@ -33,6 +43,11 @@ class PreparedReport(Document): frappe.db.set_value(self.doctype, self.name, "job_id", job_id, update_modified=False) frappe.db.commit() + def get_prepared_data(self, attachment_name=None): + if attached_file_name := attachment_name or get_attachments(self.doctype, self.name)[0]: + attached_file = frappe.get_doc("File", attached_file_name) + return gzip_decompress(attached_file.get_content()) + def generate_report(prepared_report): instance = frappe.get_doc("Prepared Report", prepared_report) @@ -54,7 +69,7 @@ def generate_report(prepared_report): report.custom_columns = data["columns"] result = generate_report_result(report=report, filters=instance.filters, user=instance.owner) - create_json_gz_file(result["result"], "Prepared Report", instance.name) + create_json_gz_file(result, instance.doctype, instance.name) instance.status = "Completed" except Exception: @@ -71,6 +86,22 @@ def generate_report(prepared_report): ) +@frappe.whitelist() +def make_prepared_report(report_name, filters=None): + """run reports in background""" + prepared_report = frappe.get_doc( + { + "doctype": "Prepared Report", + "report_name": report_name, + # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition + # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does. + "filters": json.dumps(json.loads(filters)), + } + ).insert(ignore_permissions=True) + + return {"name": prepared_report.name} + + @frappe.whitelist() def get_reports_in_queued_state(report_name, filters): reports = frappe.get_all( @@ -79,27 +110,22 @@ def get_reports_in_queued_state(report_name, filters): "report_name": report_name, "filters": json.dumps(json.loads(filters)), "status": "Queued", + "owner": frappe.session.user, }, ) return reports -def delete_expired_prepared_reports(): - system_settings = frappe.get_single("System Settings") - enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion - if enable_auto_deletion: - expiry_period = system_settings.prepared_report_expiry_period - prepared_reports_to_delete = frappe.get_all( - "Prepared Report", - filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -expiry_period)]}, - ) - - batches = frappe.utils.create_batch(prepared_reports_to_delete, 100) - for batch in batches: - args = { - "reports": batch, - } - enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args) +def get_completed_prepared_report(filters, user, report_name): + return frappe.db.get_value( + "Prepared Report", + filters={ + "status": "Completed", + "filters": json.dumps(filters), + "owner": user, + "report_name": report_name, + }, + ) @frappe.whitelist() @@ -137,7 +163,7 @@ def create_json_gz_file(data, dt, dn): @frappe.whitelist() def download_attachment(dn): attachment = get_attachments("Prepared Report", dn)[0] - frappe.local.response.filename = attachment.file_name[:-2] + frappe.local.response.filename = attachment.file_name[:-3] attached_file = frappe.get_doc("File", attachment.name) frappe.local.response.filecontent = gzip_decompress(attached_file.get_content()) frappe.local.response.type = "binary" diff --git a/frappe/core/doctype/report/report.json b/frappe/core/doctype/report/report.json index 4b98f1a3eb..37dce73dda 100644 --- a/frappe/core/doctype/report/report.json +++ b/frappe/core/doctype/report/report.json @@ -16,7 +16,6 @@ "letter_head", "add_total_row", "disabled", - "disable_prepared_report", "prepared_report", "filters_section", "filters", @@ -133,19 +132,11 @@ "label": "Roles", "options": "Has Role" }, - { - "default": "0", - "fieldname": "disable_prepared_report", - "fieldtype": "Check", - "label": "Disable Prepared Report" - }, { "default": "0", "fieldname": "prepared_report", "fieldtype": "Check", - "hidden": 1, - "label": "Prepared Report", - "read_only": 1 + "label": "Prepared Report" }, { "depends_on": "eval:(doc.report_type===\"Script Report\" \n|| doc.report_type==\"Query Report\") \n&& doc.is_standard===\"No\"", @@ -191,7 +182,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-09-15 13:37:24.531848", + "modified": "2022-11-20 14:56:36.578412", "modified_by": "Administrator", "module": "Core", "name": "Report", diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index a732a0c512..ae862184b6 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -57,17 +57,6 @@ class Report(Document): ): frappe.throw(_("You are not allowed to delete Standard Report")) delete_custom_role("report", self.name) - self.delete_prepared_reports() - - def delete_prepared_reports(self): - prepared_reports = frappe.get_all( - "Prepared Report", filters={"report_name": self.name}, pluck="name" - ) - - for report in prepared_reports: - frappe.delete_doc( - "Prepared Report", report, ignore_missing=True, force=True, delete_permanently=True - ) def get_columns(self): return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns] @@ -344,9 +333,8 @@ class Report(Document): self.db_set("disabled", cint(disable)) -@frappe.whitelist() -def is_prepared_report_disabled(report): - return frappe.db.get_value("Report", report, "disable_prepared_report") or 0 +def is_prepared_report_enabled(report): + return cint(frappe.db.get_value("Report", report, "prepared_report")) or 0 def get_report_module_dotted_path(module, report_name): diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py index bd61995ba3..17ccf35171 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py @@ -2,8 +2,9 @@ # License: MIT. See LICENSE import frappe -from frappe.core.doctype.report.report import is_prepared_report_disabled +from frappe.core.doctype.report.report import is_prepared_report_enabled from frappe.model.document import Document +from frappe.utils import cint class RolePermissionforPageandReport(Document): @@ -27,7 +28,7 @@ class RolePermissionforPageandReport(Document): def check_prepared_report_disabled(self): if self.report: - self.disable_prepared_report = is_prepared_report_disabled(self.report) + self.disable_prepared_report = not is_prepared_report_enabled(self.report) def get_standard_roles(self): doctype = self.set_role_for @@ -67,9 +68,9 @@ class RolePermissionforPageandReport(Document): if self.report: # intentionally written update query in frappe.db.sql instead of frappe.db.set_value frappe.db.sql( - """ update `tabReport` set disable_prepared_report = %s + """update `tabReport` set prepared_report = %s where name = %s""", - (self.disable_prepared_report, self.report), + (1 - cint(self.disable_prepared_report), self.report), ) def get_args(self, row=None): diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 5fd5ef8163..57e2b1a1be 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -70,9 +70,6 @@ "hide_footer_in_auto_email_reports", "attach_view_link", "prepared_report_section", - "enable_prepared_report_auto_deletion", - "prepared_report_expiry_period", - "column_break_64", "max_auto_email_report_per_user", "system_updates_section", "disable_system_update_notification", @@ -427,25 +424,11 @@ "label": "Send document Web View link in email" }, { - "default": "30", - "depends_on": "enable_prepared_report_auto_deletion", - "description": "System will auto-delete Prepared Reports permanently after these many days since creation", - "fieldname": "prepared_report_expiry_period", - "fieldtype": "Int", - "label": "Prepared Report Expiry Period (Days)" - }, - { - "default": "1", - "fieldname": "enable_prepared_report_auto_deletion", - "fieldtype": "Check", - "label": "Enable Auto-deletion of Prepared Reports" - }, - { - "collapsible": 1, - "fieldname": "prepared_report_section", - "fieldtype": "Section Break", - "label": "Reports" - }, + "collapsible": 1, + "fieldname": "prepared_report_section", + "fieldtype": "Section Break", + "label": "Reports" + }, { "default": "Frappe", "description": "The application name will be used in the Login page.", @@ -498,10 +481,6 @@ "fieldtype": "Check", "label": "Allow Older Web View Links (Insecure)" }, - { - "fieldname": "column_break_64", - "fieldtype": "Column Break" - }, { "default": "20", "fieldname": "max_auto_email_report_per_user", @@ -538,7 +517,7 @@ "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2022-10-30 12:02:46.639170", + "modified": "2022-11-20 17:57:05.099512", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 32714c2ab5..2a1ccaff98 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -145,23 +145,6 @@ def normalize_result(result, columns): return data -@frappe.whitelist() -def background_enqueue_run(report_name, filters=None): - """run reports in background""" - prepared_report = frappe.get_doc( - { - "doctype": "Prepared Report", - "report_name": report_name, - # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition - # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does. - "filters": json.dumps(json.loads(filters)), - } - ) - prepared_report.insert(ignore_permissions=True) - - return {"name": prepared_report.name} - - @frappe.whitelist() def get_script(report_name): report = get_report_doc(report_name) @@ -219,12 +202,7 @@ def run( result = None - if ( - report.prepared_report - and not report.disable_prepared_report - and not ignore_prepared_report - and not custom_columns - ): + if report.prepared_report and not ignore_prepared_report and not custom_columns: if filters: if isinstance(filters, str): filters = json.loads(filters) @@ -260,57 +238,41 @@ def add_custom_column_data(custom_columns, result): def get_prepared_report_result(report, filters, dn="", user=None): - latest_report_data = {} - doc = None - if dn: - # Get specified dn - doc = frappe.get_doc("Prepared Report", dn) - else: - # Only look for completed prepared reports with given filters. - doc_list = frappe.get_all( - "Prepared Report", - filters={ - "status": "Completed", - "filters": json.dumps(filters), - "owner": user, - "report_name": report.get("custom_report") or report.get("report_name"), - }, - order_by="creation desc", + from frappe.core.doctype.prepared_report.prepared_report import get_completed_prepared_report + + def get_report_data(doc, data): + # backwards compatibility - prepared report used to have a columns field, + # we now directly fetch it from the result file + if isinstance(data, list): + columns = (doc.get("columns") and json.loads(doc.columns)) or data[0] + result = data + else: + columns = data.get("columns") + result = data.get("result") + + for column in columns: + if isinstance(column, dict) and column.get("label"): + column["label"] = _(column["label"]) + + return {"columns": columns, "result": result} + + report_data = {} + if not dn: + dn = get_completed_prepared_report( + filters, user, report.get("custom_report") or report.get("report_name") ) - if doc_list: - # Get latest - doc = frappe.get_doc("Prepared Report", doc_list[0]) - + doc = frappe.get_doc("Prepared Report", dn) if dn else None if doc: try: - # Prepared Report data is stored in a GZip compressed JSON file - attached_file_name = frappe.db.get_value( - "File", - {"attached_to_doctype": doc.doctype, "attached_to_name": doc.name}, - "name", - ) - attached_file = frappe.get_doc("File", attached_file_name) - compressed_content = attached_file.get_content() - uncompressed_content = gzip_decompress(compressed_content) - data = json.loads(uncompressed_content.decode("utf-8")) - if data: - columns = json.loads(doc.columns) if doc.columns else data[0] - - for column in columns: - if isinstance(column, dict) and column.get("label"): - column["label"] = _(column["label"]) - - latest_report_data = {"columns": columns, "result": data} + if data := json.loads(doc.get_prepared_data().decode("utf-8")): + report_data = get_report_data(doc, data) except Exception: - doc.log_error("Prepared report failed") - frappe.delete_doc("Prepared Report", doc.name) - frappe.db.commit() + doc.log_error("Prepared report render failed") + frappe.msgprint(_("Prepared report render failed")) doc = None - latest_report_data.update({"prepared_report": True, "doc": doc}) - - return latest_report_data + return report_data | {"prepared_report": True, "doc": doc} @frappe.whitelist() diff --git a/frappe/hooks.py b/frappe/hooks.py index 73327f0ab1..ce52048c04 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -220,7 +220,6 @@ scheduler_events = { "frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry", "frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed", "frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails", - "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports", "frappe.core.doctype.log_settings.log_settings.run_log_clean_up", "frappe.utils.subscription.enable_manage_subscription", ], diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index ba7859b9a5..bcb35fad30 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -866,7 +866,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { let filters = this.get_filter_values(true); return new Promise((resolve) => frappe.call({ - method: "frappe.desk.query_report.background_enqueue_run", + method: "frappe.core.doctype.prepared_report.prepared_report.make_prepared_report", args: { report_name: this.report_name, filters: filters,