vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

refactor: cleanup peprared result render and old logs cleanup

Browse source 
* directly fetch columns from result file rather than storing it in db
* remove prepared report settings from system settings
* remove disable_prepared_report from report doctype
This commit is contained in:
phot0n 2022-11-20 18:09:41 +05:30
parent 6218a99301
commit 18d48ddeb8
10 changed files with 92 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frappe/core/doctype/log_settings/log_settings.py
View file

@ -18,6 +18,7 @@ DEFAULT_LOGTYPES_RETENTION = {
"Scheduled Job Log": 90,
"Route History": 90,
"Submission Queue": 30,
"Prepared Report": 30,
}
@ -155,7 +156,6 @@ LOG_DOCTYPES = [
"Email Queue Recipient",
"Error Snapshot",
"Error Log",
"Submission Queue",
]

5
frappe/core/doctype/prepared_report/prepared_report.json
View file

@ -22,9 +22,8 @@
"fields": [
{
"fieldname": "report_name",
"fieldtype": "Link",
"fieldtype": "Data",
"label": "Report Name",
"options": "Report",
"read_only": 1,
"reqd": 1
},
@ -99,7 +98,7 @@
],
"in_create": 1,
"links": [],
"modified": "2022-11-18 21:52:45.444209",
"modified": "2022-11-20 15:15:00.907626",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",

62
frappe/core/doctype/prepared_report/prepared_report.py
View file

@ -16,6 +16,16 @@ from frappe.utils.background_jobs import enqueue
class PreparedReport(Document):
@staticmethod
def clear_old_logs(days=30):
prepared_reports_to_delete = frappe.get_all(
"Prepared Report",
filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]},
)
for batch in frappe.utils.create_batch(prepared_reports_to_delete, 100):
enqueue(method=delete_prepared_reports, reports=batch)
def before_insert(self):
self.status = "Queued"
self.report_start_time = frappe.utils.now()
@ -33,6 +43,11 @@ class PreparedReport(Document):
frappe.db.set_value(self.doctype, self.name, "job_id", job_id, update_modified=False)
frappe.db.commit()
def get_prepared_data(self, attachment_name=None):
if attached_file_name := attachment_name or get_attachments(self.doctype, self.name)[0]:
attached_file = frappe.get_doc("File", attached_file_name)
return gzip_decompress(attached_file.get_content())
def generate_report(prepared_report):
instance = frappe.get_doc("Prepared Report", prepared_report)
@ -54,7 +69,7 @@ def generate_report(prepared_report):
report.custom_columns = data["columns"]
result = generate_report_result(report=report, filters=instance.filters, user=instance.owner)
create_json_gz_file(result["result"], "Prepared Report", instance.name)
create_json_gz_file(result, instance.doctype, instance.name)
instance.status = "Completed"
except Exception:
@ -71,6 +86,22 @@ def generate_report(prepared_report):
)
@frappe.whitelist()
def make_prepared_report(report_name, filters=None):
"""run reports in background"""
prepared_report = frappe.get_doc(
{
"doctype": "Prepared Report",
"report_name": report_name,
# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
# We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
"filters": json.dumps(json.loads(filters)),
}
).insert(ignore_permissions=True)
return {"name": prepared_report.name}
@frappe.whitelist()
def get_reports_in_queued_state(report_name, filters):
reports = frappe.get_all(
@ -79,27 +110,22 @@ def get_reports_in_queued_state(report_name, filters):
"report_name": report_name,
"filters": json.dumps(json.loads(filters)),
"status": "Queued",
"owner": frappe.session.user,
},
)
return reports
def delete_expired_prepared_reports():
system_settings = frappe.get_single("System Settings")
enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion
if enable_auto_deletion:
expiry_period = system_settings.prepared_report_expiry_period
prepared_reports_to_delete = frappe.get_all(
"Prepared Report",
filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -expiry_period)]},
)
batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
for batch in batches:
args = {
"reports": batch,
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
def get_completed_prepared_report(filters, user, report_name):
return frappe.db.get_value(
"Prepared Report",
filters={
"status": "Completed",
"filters": json.dumps(filters),
"owner": user,
"report_name": report_name,
},
)
@frappe.whitelist()
@ -137,7 +163,7 @@ def create_json_gz_file(data, dt, dn):
@frappe.whitelist()
def download_attachment(dn):
attachment = get_attachments("Prepared Report", dn)[0]
frappe.local.response.filename = attachment.file_name[:-2]
frappe.local.response.filename = attachment.file_name[:-3]
attached_file = frappe.get_doc("File", attachment.name)
frappe.local.response.filecontent = gzip_decompress(attached_file.get_content())
frappe.local.response.type = "binary"

13
frappe/core/doctype/report/report.json
View file

@ -16,7 +16,6 @@
"letter_head",
"add_total_row",
"disabled",
"disable_prepared_report",
"prepared_report",
"filters_section",
"filters",
@ -133,19 +132,11 @@
"label": "Roles",
"options": "Has Role"
},
{
"default": "0",
"fieldname": "disable_prepared_report",
"fieldtype": "Check",
"label": "Disable Prepared Report"
},
{
"default": "0",
"fieldname": "prepared_report",
"fieldtype": "Check",
"hidden": 1,
"label": "Prepared Report",
"read_only": 1
"label": "Prepared Report"
},
{
"depends_on": "eval:(doc.report_type===\"Script Report\" \n|| doc.report_type==\"Query Report\") \n&& doc.is_standard===\"No\"",
@ -191,7 +182,7 @@
"idx": 1,
"index_web_pages_for_search": 1,
"links": [],
"modified": "2022-09-15 13:37:24.531848",
"modified": "2022-11-20 14:56:36.578412",
"modified_by": "Administrator",
"module": "Core",
"name": "Report",

16
frappe/core/doctype/report/report.py
View file

@ -57,17 +57,6 @@ class Report(Document):
):
frappe.throw(_("You are not allowed to delete Standard Report"))
delete_custom_role("report", self.name)
self.delete_prepared_reports()
def delete_prepared_reports(self):
prepared_reports = frappe.get_all(
"Prepared Report", filters={"report_name": self.name}, pluck="name"
)
for report in prepared_reports:
frappe.delete_doc(
"Prepared Report", report, ignore_missing=True, force=True, delete_permanently=True
)
def get_columns(self):
return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
@ -344,9 +333,8 @@ class Report(Document):
self.db_set("disabled", cint(disable))
@frappe.whitelist()
def is_prepared_report_disabled(report):
return frappe.db.get_value("Report", report, "disable_prepared_report") or 0
def is_prepared_report_enabled(report):
return cint(frappe.db.get_value("Report", report, "prepared_report")) or 0
def get_report_module_dotted_path(module, report_name):

9
frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
View file

@ -2,8 +2,9 @@
# License: MIT. See LICENSE
import frappe
from frappe.core.doctype.report.report import is_prepared_report_disabled
from frappe.core.doctype.report.report import is_prepared_report_enabled
from frappe.model.document import Document
from frappe.utils import cint
class RolePermissionforPageandReport(Document):
@ -27,7 +28,7 @@ class RolePermissionforPageandReport(Document):
def check_prepared_report_disabled(self):
if self.report:
self.disable_prepared_report = is_prepared_report_disabled(self.report)
self.disable_prepared_report = not is_prepared_report_enabled(self.report)
def get_standard_roles(self):
doctype = self.set_role_for
@ -67,9 +68,9 @@ class RolePermissionforPageandReport(Document):
if self.report:
# intentionally written update query in frappe.db.sql instead of frappe.db.set_value
frappe.db.sql(
""" update `tabReport` set disable_prepared_report = %s
"""update `tabReport` set prepared_report = %s
where name = %s""",
(self.disable_prepared_report, self.report),
(1 - cint(self.disable_prepared_report), self.report),
)
def get_args(self, row=None):

33
frappe/core/doctype/system_settings/system_settings.json
View file

@ -70,9 +70,6 @@
"hide_footer_in_auto_email_reports",
"attach_view_link",
"prepared_report_section",
"enable_prepared_report_auto_deletion",
"prepared_report_expiry_period",
"column_break_64",
"max_auto_email_report_per_user",
"system_updates_section",
"disable_system_update_notification",
@ -427,25 +424,11 @@
"label": "Send document Web View link in email"
},
{
"default": "30",
"depends_on": "enable_prepared_report_auto_deletion",
"description": "System will auto-delete Prepared Reports permanently after these many days since creation",
"fieldname": "prepared_report_expiry_period",
"fieldtype": "Int",
"label": "Prepared Report Expiry Period (Days)"
},
{
"default": "1",
"fieldname": "enable_prepared_report_auto_deletion",
"fieldtype": "Check",
"label": "Enable Auto-deletion of Prepared Reports"
},
{
"collapsible": 1,
"fieldname": "prepared_report_section",
"fieldtype": "Section Break",
"label": "Reports"
},
"collapsible": 1,
"fieldname": "prepared_report_section",
"fieldtype": "Section Break",
"label": "Reports"
},
{
"default": "Frappe",
"description": "The application name will be used in the Login page.",
@ -498,10 +481,6 @@
"fieldtype": "Check",
"label": "Allow Older Web View Links (Insecure)"
},
{
"fieldname": "column_break_64",
"fieldtype": "Column Break"
},
{
"default": "20",
"fieldname": "max_auto_email_report_per_user",
@ -538,7 +517,7 @@
"icon": "fa fa-cog",
"issingle": 1,
"links": [],
"modified": "2022-10-30 12:02:46.639170",
"modified": "2022-11-20 17:57:05.099512",
"modified_by": "Administrator",
"module": "Core",
"name": "System Settings",

96
frappe/desk/query_report.py
View file

@ -145,23 +145,6 @@ def normalize_result(result, columns):
return data
@frappe.whitelist()
def background_enqueue_run(report_name, filters=None):
"""run reports in background"""
prepared_report = frappe.get_doc(
{
"doctype": "Prepared Report",
"report_name": report_name,
# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
# We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
"filters": json.dumps(json.loads(filters)),
}
)
prepared_report.insert(ignore_permissions=True)
return {"name": prepared_report.name}
@frappe.whitelist()
def get_script(report_name):
report = get_report_doc(report_name)
@ -219,12 +202,7 @@ def run(
result = None
if (
report.prepared_report
and not report.disable_prepared_report
and not ignore_prepared_report
and not custom_columns
):
if report.prepared_report and not ignore_prepared_report and not custom_columns:
if filters:
if isinstance(filters, str):
filters = json.loads(filters)
@ -260,57 +238,41 @@ def add_custom_column_data(custom_columns, result):
def get_prepared_report_result(report, filters, dn="", user=None):
latest_report_data = {}
doc = None
if dn:
# Get specified dn
doc = frappe.get_doc("Prepared Report", dn)
else:
# Only look for completed prepared reports with given filters.
doc_list = frappe.get_all(
"Prepared Report",
filters={
"status": "Completed",
"filters": json.dumps(filters),
"owner": user,
"report_name": report.get("custom_report") or report.get("report_name"),
},
order_by="creation desc",
from frappe.core.doctype.prepared_report.prepared_report import get_completed_prepared_report
def get_report_data(doc, data):
# backwards compatibility - prepared report used to have a columns field,
# we now directly fetch it from the result file
if isinstance(data, list):
columns = (doc.get("columns") and json.loads(doc.columns)) or data[0]
result = data
else:
columns = data.get("columns")
result = data.get("result")
for column in columns:
if isinstance(column, dict) and column.get("label"):
column["label"] = _(column["label"])
return {"columns": columns, "result": result}
report_data = {}
if not dn:
dn = get_completed_prepared_report(
filters, user, report.get("custom_report") or report.get("report_name")
)
if doc_list:
# Get latest
doc = frappe.get_doc("Prepared Report", doc_list[0])
doc = frappe.get_doc("Prepared Report", dn) if dn else None
if doc:
try:
# Prepared Report data is stored in a GZip compressed JSON file
attached_file_name = frappe.db.get_value(
"File",
{"attached_to_doctype": doc.doctype, "attached_to_name": doc.name},
"name",
)
attached_file = frappe.get_doc("File", attached_file_name)
compressed_content = attached_file.get_content()
uncompressed_content = gzip_decompress(compressed_content)
data = json.loads(uncompressed_content.decode("utf-8"))
if data:
columns = json.loads(doc.columns) if doc.columns else data[0]
for column in columns:
if isinstance(column, dict) and column.get("label"):
column["label"] = _(column["label"])
latest_report_data = {"columns": columns, "result": data}
if data := json.loads(doc.get_prepared_data().decode("utf-8")):
report_data = get_report_data(doc, data)
except Exception:
doc.log_error("Prepared report failed")
frappe.delete_doc("Prepared Report", doc.name)
frappe.db.commit()
doc.log_error("Prepared report render failed")
frappe.msgprint(_("Prepared report render failed"))
doc = None
latest_report_data.update({"prepared_report": True, "doc": doc})
return latest_report_data
return report_data | {"prepared_report": True, "doc": doc}
@frappe.whitelist()

1
frappe/hooks.py
View file

@ -220,7 +220,6 @@ scheduler_events = {
"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
"frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports",
"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",
"frappe.utils.subscription.enable_manage_subscription",
],

2
frappe/public/js/frappe/views/reports/query_report.js
View file

@ -866,7 +866,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
let filters = this.get_filter_values(true);
return new Promise((resolve) =>
frappe.call({
method: "frappe.desk.query_report.background_enqueue_run",
method: "frappe.core.doctype.prepared_report.prepared_report.make_prepared_report",
args: {
report_name: this.report_name,
filters: filters,