diff --git a/frappe/__init__.py b/frappe/__init__.py
index 2d491ca068..8424fd0071 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1987,6 +1987,7 @@ def get_print(
no_letterhead=0,
password=None,
pdf_options=None,
+ letterhead=None,
):
"""Get Print Format for given document.
@@ -2005,6 +2006,7 @@ def get_print(
local.form_dict.style = style
local.form_dict.doc = doc
local.form_dict.no_letterhead = no_letterhead
+ local.form_dict.letterhead = letterhead
pdf_options = pdf_options or {}
if password:
diff --git a/frappe/app.py b/frappe/app.py
index fc679aa44e..2fe9991c4c 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -77,7 +77,7 @@ def application(request: Request):
rollback = after_request(rollback)
finally:
- if request.method in ("POST", "PUT") and frappe.db and rollback:
+ if request.method in UNSAFE_HTTP_METHODS and frappe.db and rollback:
frappe.db.rollback()
frappe.rate_limiter.update()
@@ -320,12 +320,16 @@ def handle_exception(e):
def after_request(rollback):
# if HTTP method would change server state, commit if necessary
- if frappe.db and (
- frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS
+ if (
+ frappe.db
+ and (frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS)
+ and frappe.db.transaction_writes
):
- if frappe.db.transaction_writes:
- frappe.db.commit()
- rollback = False
+ frappe.db.commit()
+ rollback = False
+ elif frappe.db:
+ frappe.db.rollback()
+ rollback = False
# update session
if getattr(frappe.local, "session_obj", None):
diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py
index e481676088..0c8278dcbf 100755
--- a/frappe/commands/scheduler.py
+++ b/frappe/commands/scheduler.py
@@ -178,15 +178,34 @@ def start_scheduler():
@click.command("worker")
-@click.option("--queue", type=str)
+@click.option(
+ "--queue",
+ type=str,
+ help="Queue to consume from. Multiple queues can be specified using comma-separated string. If not specified all queues are consumed.",
+)
@click.option("--quiet", is_flag=True, default=False, help="Hide Log Outputs")
@click.option("-u", "--rq-username", default=None, help="Redis ACL user")
@click.option("-p", "--rq-password", default=None, help="Redis ACL user password")
-def start_worker(queue, quiet=False, rq_username=None, rq_password=None):
- """Site is used to find redis credentals."""
+@click.option("--burst", is_flag=True, default=False, help="Run Worker in Burst mode.")
+@click.option(
+ "--strategy",
+ required=False,
+ type=click.Choice(["round_robin", "random"]),
+ help="Dequeuing strategy to use",
+)
+def start_worker(
+ queue, quiet=False, rq_username=None, rq_password=None, burst=False, strategy=None
+):
from frappe.utils.background_jobs import start_worker
- start_worker(queue, quiet=quiet, rq_username=rq_username, rq_password=rq_password)
+ start_worker(
+ queue,
+ quiet=quiet,
+ rq_username=rq_username,
+ rq_password=rq_password,
+ burst=burst,
+ strategy=strategy,
+ )
@click.command("ready-for-migration")
diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 13f642ea76..37b1a36c6c 100644
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -1119,7 +1119,7 @@ def build_search_index(context):
@click.command("clear-log-table")
-@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType")
+@click.option("--doctype", required=True, type=click.Choice(LOG_DOCTYPES), help="Log DocType")
@click.option("--days", type=int, help="Keep records for days")
@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table")
@pass_context
diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py
index 611592531d..366245b5e8 100644
--- a/frappe/core/doctype/data_export/exporter.py
+++ b/frappe/core/doctype/data_export/exporter.py
@@ -432,23 +432,20 @@ class DataExporter:
row[_column_start_end.start + i + 1] = value
def build_response_as_excel(self):
+ from frappe.desk.utils import provide_binary_file
+ from frappe.utils.xlsxutils import make_xlsx
+
filename = frappe.generate_hash(length=10)
with open(filename, "wb") as f:
f.write(cstr(self.writer.getvalue()).encode("utf-8"))
f = open(filename)
reader = csv.reader(f)
-
- from frappe.utils.xlsxutils import make_xlsx
-
xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else "Data Export")
f.close()
os.remove(filename)
- # write out response as a xlsx type
- frappe.response["filename"] = self.doctype + ".xlsx"
- frappe.response["filecontent"] = xlsx_file.getvalue()
- frappe.response["type"] = "binary"
+ provide_binary_file(self.doctype, "xlsx", xlsx_file.getvalue())
def _append_name_column(self, dt=None):
self.append_field_column(
diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py
index 9c793767e3..1066b8bed9 100644
--- a/frappe/core/doctype/data_import/exporter.py
+++ b/frappe/core/doctype/data_import/exporter.py
@@ -1,8 +1,6 @@
# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
-import typing
-
import frappe
from frappe import _
from frappe.model import display_fieldtypes, no_value_fields
@@ -241,15 +239,9 @@ class Exporter:
def build_response(self):
if self.file_type == "CSV":
- self.build_csv_response()
+ build_csv_response(self.get_csv_array_for_export(), _(self.doctype))
elif self.file_type == "Excel":
- self.build_xlsx_response()
-
- def build_csv_response(self):
- build_csv_response(self.get_csv_array_for_export(), _(self.doctype))
-
- def build_xlsx_response(self):
- build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype))
+ build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype))
def group_children_data_by_parent(self, children_data: dict[str, list]):
return groupby_metric(children_data, key="parent")
diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js
index 24c367b115..7324a92359 100644
--- a/frappe/core/doctype/doctype/doctype.js
+++ b/frappe/core/doctype/doctype/doctype.js
@@ -124,6 +124,7 @@ frappe.ui.form.on("DocField", {
let doctypes = frm.doc.fields
.filter((df) => df.fieldtype == "Link")
.filter((df) => df.options && df.fieldname != row.fieldname)
+ .sort((a, b) => a.options.localeCompare(b.options))
.map((df) => ({
label: `${df.options} (${df.fieldname})`,
value: df.fieldname,
@@ -151,6 +152,7 @@ frappe.ui.form.on("DocField", {
.get_docfields(link_doctype, null, {
fieldtype: ["not in", frappe.model.no_value_type],
})
+ .sort((a, b) => a.label.localeCompare(b.label))
.map((df) => ({
label: `${df.label} (${df.fieldtype})`,
value: df.fieldname,
diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py
index f0f2cdaae8..e1e1d5d176 100644
--- a/frappe/core/doctype/log_settings/log_settings.py
+++ b/frappe/core/doctype/log_settings/log_settings.py
@@ -18,6 +18,7 @@ DEFAULT_LOGTYPES_RETENTION = {
"Scheduled Job Log": 90,
"Route History": 90,
"Submission Queue": 30,
+ "Prepared Report": 30,
}
@@ -155,7 +156,6 @@ LOG_DOCTYPES = [
"Email Queue Recipient",
"Error Snapshot",
"Error Log",
- "Submission Queue",
]
diff --git a/frappe/core/doctype/prepared_report/prepared_report.js b/frappe/core/doctype/prepared_report/prepared_report.js
index 58f4c1957f..11f9caf8fd 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.js
+++ b/frappe/core/doctype/prepared_report/prepared_report.js
@@ -2,7 +2,7 @@
// For license information, please see license.txt
frappe.ui.form.on("Prepared Report", {
- onload: function (frm) {
+ render_filter_values: function (frm) {
var wrapper = $(frm.fields_dict["filter_values"].wrapper).empty();
let filter_table = $(`
@@ -16,6 +16,7 @@ frappe.ui.form.on("Prepared Report", {
`);
const filters = JSON.parse(frm.doc.filters);
+ frm.toggle_display(["filter_values"], !$.isEmptyObject(filters));
Object.keys(filters).forEach((key) => {
const filter_row = $(`
@@ -30,6 +31,12 @@ frappe.ui.form.on("Prepared Report", {
refresh: function (frm) {
frm.disable_save();
+ frm.events.render_filter_values(frm);
+
+ // always keep report_name hidden - we do this as we can't set mandatory and hidden
+ // property on a docfield at the same time
+ frm.toggle_display(["report_name"], 0);
+
if (frm.doc.status == "Completed") {
frm.page.set_primary_action(__("Show Report"), () => {
frappe.set_route(
diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json
index cafe323519..d00175b693 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.json
+++ b/frappe/core/doctype/prepared_report/prepared_report.json
@@ -1,38 +1,31 @@
{
"actions": [],
- "autoname": "REP.#####",
+ "autoname": "hash",
"creation": "2018-06-25 18:39:11.152960",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
- "report_name",
- "ref_report_doctype",
"status",
+ "report_name",
+ "queued_by",
+ "job_id",
"column_break_4",
- "report_start_time",
+ "queued_at",
"report_end_time",
"section_break_7",
"error_message",
"filters_sb",
"filters",
- "filter_values",
- "columns"
+ "filter_values"
],
"fields": [
{
"fieldname": "report_name",
"fieldtype": "Data",
"label": "Report Name",
- "read_only": 1
- },
- {
- "fieldname": "ref_report_doctype",
- "fieldtype": "Link",
- "in_standard_filter": 1,
- "label": "Report Type",
- "options": "Report",
- "read_only": 1
+ "read_only": 1,
+ "reqd": 1
},
{
"default": "Queued",
@@ -49,16 +42,10 @@
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
- {
- "fieldname": "report_start_time",
- "fieldtype": "Datetime",
- "label": "Report Start Time",
- "read_only": 1
- },
{
"fieldname": "report_end_time",
"fieldtype": "Datetime",
- "label": "Report End Time",
+ "label": "Finished At",
"read_only": 1
},
{
@@ -92,22 +79,35 @@
"label": "Filter Values"
},
{
- "fieldname": "columns",
- "fieldtype": "Code",
- "hidden": 1,
- "label": "Columns",
+ "fieldname": "job_id",
+ "fieldtype": "Link",
+ "label": "Job ID",
"no_copy": 1,
- "print_hide": 1,
+ "options": "RQ Job",
+ "read_only": 1
+ },
+ {
+ "fieldname": "queued_by",
+ "fieldtype": "Data",
+ "is_virtual": 1,
+ "label": "Queued By",
+ "read_only": 1
+ },
+ {
+ "fieldname": "queued_at",
+ "fieldtype": "Datetime",
+ "is_virtual": 1,
+ "label": "Queued At",
"read_only": 1
}
],
"in_create": 1,
"links": [],
- "modified": "2022-06-13 06:20:34.496412",
+ "modified": "2022-11-28 21:29:39.883803",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
- "naming_rule": "Expression (old style)",
+ "naming_rule": "Random",
"owner": "Administrator",
"permissions": [
{
@@ -134,7 +134,20 @@
],
"sort_field": "modified",
"sort_order": "DESC",
- "states": [],
- "title_field": "ref_report_doctype",
+ "states": [
+ {
+ "color": "Blue",
+ "title": "Queued"
+ },
+ {
+ "color": "Red",
+ "title": "Error"
+ },
+ {
+ "color": "Green",
+ "title": "Completed"
+ }
+ ],
+ "title_field": "report_name",
"track_changes": 1
}
\ No newline at end of file
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 0b8e25229d..b5036aaf67 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -4,6 +4,8 @@
import json
+from rq import get_current_job
+
import frappe
from frappe.desk.form.load import get_attachments
from frappe.desk.query_report import generate_report_result
@@ -14,19 +16,53 @@ from frappe.utils.background_jobs import enqueue
class PreparedReport(Document):
+ @property
+ def queued_by(self):
+ return self.owner
+
+ @property
+ def queued_at(self):
+ return self.creation
+
+ @staticmethod
+ def clear_old_logs(days=30):
+ prepared_reports_to_delete = frappe.get_all(
+ "Prepared Report",
+ filters={"modified": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]},
+ )
+
+ for batch in frappe.utils.create_batch(prepared_reports_to_delete, 100):
+ enqueue(method=delete_prepared_reports, reports=batch)
+
def before_insert(self):
self.status = "Queued"
- self.report_start_time = frappe.utils.now()
- def enqueue_report(self):
- enqueue(run_background, prepared_report=self.name, timeout=6000)
+ def after_insert(self):
+ enqueue(
+ generate_report,
+ queue="long",
+ prepared_report=self.name,
+ timeout=1500,
+ enqueue_after_commit=True,
+ )
+
+ def get_prepared_data(self, with_file_name=False):
+ if attachments := get_attachments(self.doctype, self.name):
+ attachment = attachments[0]
+ attached_file = frappe.get_doc("File", attachment.name)
+
+ if with_file_name:
+ return (gzip_decompress(attached_file.get_content()), attachment.file_name)
+ return gzip_decompress(attached_file.get_content())
-def run_background(prepared_report):
+def generate_report(prepared_report):
+ update_job_id(prepared_report, get_current_job().id)
+
instance = frappe.get_doc("Prepared Report", prepared_report)
- report = frappe.get_doc("Report", instance.ref_report_doctype)
+ report = frappe.get_doc("Report", instance.report_name)
- add_data_to_monitor(report=instance.ref_report_doctype)
+ add_data_to_monitor(report=instance.report_name)
try:
report.custom_columns = []
@@ -41,19 +77,15 @@ def run_background(prepared_report):
report.custom_columns = data["columns"]
result = generate_report_result(report=report, filters=instance.filters, user=instance.owner)
- create_json_gz_file(result["result"], "Prepared Report", instance.name)
+ create_json_gz_file(result, instance.doctype, instance.name)
instance.status = "Completed"
- instance.columns = json.dumps(result["columns"])
- instance.report_end_time = frappe.utils.now()
- instance.save(ignore_permissions=True)
-
except Exception:
- report.log_error("Prepared report failed")
- instance = frappe.get_doc("Prepared Report", prepared_report)
instance.status = "Error"
instance.error_message = frappe.get_traceback()
- instance.save(ignore_permissions=True)
+
+ instance.report_end_time = frappe.utils.now()
+ instance.save(ignore_permissions=True)
frappe.publish_realtime(
"report_generated",
@@ -62,6 +94,27 @@ def run_background(prepared_report):
)
+def update_job_id(prepared_report, job_id):
+ frappe.db.set_value("Prepared Report", prepared_report, "job_id", job_id, update_modified=False)
+ frappe.db.commit()
+
+
+@frappe.whitelist()
+def make_prepared_report(report_name, filters=None):
+ """run reports in background"""
+ prepared_report = frappe.get_doc(
+ {
+ "doctype": "Prepared Report",
+ "report_name": report_name,
+ # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
+ # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
+ "filters": json.dumps(json.loads(filters)),
+ }
+ ).insert(ignore_permissions=True)
+
+ return {"name": prepared_report.name}
+
+
@frappe.whitelist()
def get_reports_in_queued_state(report_name, filters):
reports = frappe.get_all(
@@ -70,27 +123,22 @@ def get_reports_in_queued_state(report_name, filters):
"report_name": report_name,
"filters": json.dumps(json.loads(filters)),
"status": "Queued",
+ "owner": frappe.session.user,
},
)
return reports
-def delete_expired_prepared_reports():
- system_settings = frappe.get_single("System Settings")
- enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion
- if enable_auto_deletion:
- expiry_period = system_settings.prepared_report_expiry_period
- prepared_reports_to_delete = frappe.get_all(
- "Prepared Report",
- filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -expiry_period)]},
- )
-
- batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
- for batch in batches:
- args = {
- "reports": batch,
- }
- enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
+def get_completed_prepared_report(filters, user, report_name):
+ return frappe.db.get_value(
+ "Prepared Report",
+ filters={
+ "status": "Completed",
+ "filters": json.dumps(filters),
+ "owner": user,
+ "report_name": report_name,
+ },
+ )
@frappe.whitelist()
@@ -127,10 +175,13 @@ def create_json_gz_file(data, dt, dn):
@frappe.whitelist()
def download_attachment(dn):
- attachment = get_attachments("Prepared Report", dn)[0]
- frappe.local.response.filename = attachment.file_name[:-2]
- attached_file = frappe.get_doc("File", attachment.name)
- frappe.local.response.filecontent = gzip_decompress(attached_file.get_content())
+ pr = frappe.get_doc("Prepared Report", dn)
+ if not pr.has_permission("read"):
+ frappe.throw(frappe._("Cannot Download Report due to insufficient permissions"))
+
+ data, file_name = pr.get_prepared_data(with_file_name=True)
+ frappe.local.response.filename = file_name[:-3]
+ frappe.local.response.filecontent = data
frappe.local.response.type = "binary"
@@ -149,9 +200,7 @@ def get_permission_query_condition(user):
reports = [frappe.db.escape(report) for report in user.get_all_reports().keys()]
- return """`tabPrepared Report`.ref_report_doctype in ({reports})""".format(
- reports=",".join(reports)
- )
+ return """`tabPrepared Report`.report_name in ({reports})""".format(reports=",".join(reports))
def has_permission(doc, user):
@@ -167,4 +216,4 @@ def has_permission(doc, user):
if "System Manager" in user.roles:
return True
- return doc.ref_report_doctype in user.get_all_reports().keys()
+ return doc.report_name in user.get_all_reports().keys()
diff --git a/frappe/core/doctype/prepared_report/prepared_report_list.js b/frappe/core/doctype/prepared_report/prepared_report_list.js
deleted file mode 100644
index 1414dd7580..0000000000
--- a/frappe/core/doctype/prepared_report/prepared_report_list.js
+++ /dev/null
@@ -1,12 +0,0 @@
-frappe.listview_settings["Prepared Report"] = {
- add_fields: ["status"],
- get_indicator: function (doc) {
- if (doc.status === "Completed") {
- return [__("Completed"), "green", "status,=,Completed"];
- } else if (doc.status === "Error") {
- return [__("Error"), "red", "status,=,Error"];
- } else if (doc.status === "Queued") {
- return [__("Queued"), "orange", "status,=,Queued"];
- }
- },
-};
diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py
index cd96e8a18f..a864ea73f8 100644
--- a/frappe/core/doctype/prepared_report/test_prepared_report.py
+++ b/frappe/core/doctype/prepared_report/test_prepared_report.py
@@ -1,28 +1,53 @@
# Copyright (c) 2018, Frappe Technologies and Contributors
# License: MIT. See LICENSE
import json
+import time
import frappe
+from frappe.desk.query_report import generate_report_result, get_report_doc
from frappe.tests.utils import FrappeTestCase
class TestPreparedReport(FrappeTestCase):
- def setUp(self):
- self.report = frappe.get_doc({"doctype": "Report", "name": "Permitted Documents For User"})
- self.filters = {"user": "Administrator", "doctype": "Role"}
- self.prepared_report_doc = frappe.get_doc(
+ @classmethod
+ def tearDownClass(cls):
+ for r in frappe.get_all("Prepared Report", pluck="name"):
+ frappe.delete_doc("Prepared Report", r, force=True, delete_permanently=True)
+
+ frappe.db.commit()
+
+ def create_prepared_report(self, commit=False):
+ doc = frappe.get_doc(
{
"doctype": "Prepared Report",
- "report_name": self.report.name,
- "filters": json.dumps(self.filters),
- "ref_report_doctype": self.report.name,
+ "report_name": "Database Storage Usage By Tables",
}
).insert()
- def tearDown(self):
- frappe.set_user("Administrator")
- self.prepared_report_doc.delete()
+ if commit:
+ frappe.db.commit()
- def test_for_creation(self):
- self.assertTrue("QUEUED" == self.prepared_report_doc.status.upper())
- self.assertTrue(self.prepared_report_doc.report_start_time)
+ return doc
+
+ def test_queueing(self):
+ doc_ = self.create_prepared_report()
+ self.assertEqual("Queued", doc_.status)
+ self.assertTrue(doc_.queued_at)
+
+ frappe.db.commit()
+ time.sleep(5)
+
+ doc_ = frappe.get_last_doc("Prepared Report")
+ self.assertEqual("Completed", doc_.status)
+ self.assertTrue(doc_.job_id)
+ self.assertTrue(doc_.report_end_time)
+
+ def test_prepared_data(self):
+ doc_ = self.create_prepared_report(commit=True)
+ time.sleep(5)
+
+ prepared_data = json.loads(doc_.get_prepared_data().decode("utf-8"))
+ generated_data = generate_report_result(get_report_doc("Database Storage Usage By Tables"))
+ self.assertEqual(len(prepared_data["columns"]), len(generated_data["columns"]))
+ self.assertEqual(len(prepared_data["result"]), len(generated_data["result"]))
+ self.assertEqual(len(prepared_data), len(generated_data))
diff --git a/frappe/core/doctype/report/report.json b/frappe/core/doctype/report/report.json
index 4b98f1a3eb..37dce73dda 100644
--- a/frappe/core/doctype/report/report.json
+++ b/frappe/core/doctype/report/report.json
@@ -16,7 +16,6 @@
"letter_head",
"add_total_row",
"disabled",
- "disable_prepared_report",
"prepared_report",
"filters_section",
"filters",
@@ -133,19 +132,11 @@
"label": "Roles",
"options": "Has Role"
},
- {
- "default": "0",
- "fieldname": "disable_prepared_report",
- "fieldtype": "Check",
- "label": "Disable Prepared Report"
- },
{
"default": "0",
"fieldname": "prepared_report",
"fieldtype": "Check",
- "hidden": 1,
- "label": "Prepared Report",
- "read_only": 1
+ "label": "Prepared Report"
},
{
"depends_on": "eval:(doc.report_type===\"Script Report\" \n|| doc.report_type==\"Query Report\") \n&& doc.is_standard===\"No\"",
@@ -191,7 +182,7 @@
"idx": 1,
"index_web_pages_for_search": 1,
"links": [],
- "modified": "2022-09-15 13:37:24.531848",
+ "modified": "2022-11-20 14:56:36.578412",
"modified_by": "Administrator",
"module": "Core",
"name": "Report",
diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 7fe3cadf9c..ae862184b6 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -57,17 +57,6 @@ class Report(Document):
):
frappe.throw(_("You are not allowed to delete Standard Report"))
delete_custom_role("report", self.name)
- self.delete_prepared_reports()
-
- def delete_prepared_reports(self):
- prepared_reports = frappe.get_all(
- "Prepared Report", filters={"ref_report_doctype": self.name}, pluck="name"
- )
-
- for report in prepared_reports:
- frappe.delete_doc(
- "Prepared Report", report, ignore_missing=True, force=True, delete_permanently=True
- )
def get_columns(self):
return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
@@ -344,9 +333,8 @@ class Report(Document):
self.db_set("disabled", cint(disable))
-@frappe.whitelist()
-def is_prepared_report_disabled(report):
- return frappe.db.get_value("Report", report, "disable_prepared_report") or 0
+def is_prepared_report_enabled(report):
+ return cint(frappe.db.get_value("Report", report, "prepared_report")) or 0
def get_report_module_dotted_path(module, report_name):
diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json
index 09982cf639..52ecc5d38f 100644
--- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json
+++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json
@@ -10,7 +10,7 @@
"page",
"report",
"column_break_4",
- "disable_prepared_report",
+ "enable_prepared_report",
"roles_permission",
"roles_html",
"roles"
@@ -42,13 +42,6 @@
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
- {
- "default": "0",
- "depends_on": "report",
- "fieldname": "disable_prepared_report",
- "fieldtype": "Check",
- "label": "Disable Prepared Report"
- },
{
"fieldname": "roles_permission",
"fieldtype": "Section Break",
@@ -66,12 +59,19 @@
"label": "Roles",
"options": "Has Role",
"read_only": 1
+ },
+ {
+ "default": "0",
+ "depends_on": "report",
+ "fieldname": "enable_prepared_report",
+ "fieldtype": "Check",
+ "label": "Enable Prepared Report"
}
],
"hide_toolbar": 1,
"issingle": 1,
"links": [],
- "modified": "2022-08-03 12:20:54.079809",
+ "modified": "2022-11-23 12:39:05.750386",
"modified_by": "Administrator",
"module": "Core",
"name": "Role Permission for Page and Report",
diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
index bd61995ba3..9a3511184d 100644
--- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
+++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
@@ -2,8 +2,9 @@
# License: MIT. See LICENSE
import frappe
-from frappe.core.doctype.report.report import is_prepared_report_disabled
+from frappe.core.doctype.report.report import is_prepared_report_enabled
from frappe.model.document import Document
+from frappe.utils import cint
class RolePermissionforPageandReport(Document):
@@ -27,7 +28,7 @@ class RolePermissionforPageandReport(Document):
def check_prepared_report_disabled(self):
if self.report:
- self.disable_prepared_report = is_prepared_report_disabled(self.report)
+ self.enable_prepared_report = is_prepared_report_enabled(self.report)
def get_standard_roles(self):
doctype = self.set_role_for
@@ -67,9 +68,9 @@ class RolePermissionforPageandReport(Document):
if self.report:
# intentionally written update query in frappe.db.sql instead of frappe.db.set_value
frappe.db.sql(
- """ update `tabReport` set disable_prepared_report = %s
+ """update `tabReport` set prepared_report = %s
where name = %s""",
- (self.disable_prepared_report, self.report),
+ (self.enable_prepared_report, self.report),
)
def get_args(self, row=None):
diff --git a/frappe/core/doctype/role_permission_for_page_and_report/test_role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/test_role_permission_for_page_and_report.py
new file mode 100644
index 0000000000..b05da325e6
--- /dev/null
+++ b/frappe/core/doctype/role_permission_for_page_and_report/test_role_permission_for_page_and_report.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2022, Frappe Technologies and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestRolePermissionforPageandReport(FrappeTestCase):
+ pass
diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py
index 460aa08941..654cdcd21f 100644
--- a/frappe/core/doctype/rq_job/test_rq_job.py
+++ b/frappe/core/doctype/rq_job/test_rq_job.py
@@ -10,6 +10,7 @@ from rq.job import Job
import frappe
from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs, stop_job
from frappe.tests.utils import FrappeTestCase, timeout
+from frappe.utils import cstr, execute_in_shell
from frappe.utils.background_jobs import is_job_queued
@@ -92,6 +93,15 @@ class TestRQJob(FrappeTestCase):
self.check_status(actual_job, "finished")
self.assertFalse(is_job_queued(job_name))
+ @timeout(20)
+ def test_multi_queue_burst_consumption(self):
+ for _ in range(3):
+ for q in ["default", "short"]:
+ frappe.enqueue(self.BG_JOB, sleep=1, queue=q)
+
+ _, stderr = execute_in_shell("bench worker --queue short,default --burst", check_exit_code=True)
+ self.assertIn("quitting", cstr(stderr))
+
def test_func(fail=False, sleep=0):
if fail:
diff --git a/frappe/core/doctype/rq_worker/rq_worker.json b/frappe/core/doctype/rq_worker/rq_worker.json
index d9a5a23f67..18441377c9 100644
--- a/frappe/core/doctype/rq_worker/rq_worker.json
+++ b/frappe/core/doctype/rq_worker/rq_worker.json
@@ -76,13 +76,13 @@
{
"fieldname": "queue",
"fieldtype": "Data",
- "label": "Queue"
+ "label": "Queue(s)"
},
{
"fieldname": "queue_type",
"fieldtype": "Select",
"in_list_view": 1,
- "label": "Queue Type",
+ "label": "Queue Type(s)",
"options": "default\nlong\nshort"
},
{
@@ -113,7 +113,7 @@
"in_create": 1,
"is_virtual": 1,
"links": [],
- "modified": "2022-11-14 15:35:32.786012",
+ "modified": "2022-11-24 14:50:48.511706",
"modified_by": "Administrator",
"module": "Core",
"name": "RQ Worker",
diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py
index 3de0c8f7fc..1d24001fc3 100644
--- a/frappe/core/doctype/rq_worker/rq_worker.py
+++ b/frappe/core/doctype/rq_worker/rq_worker.py
@@ -16,8 +16,10 @@ class RQWorker(Document):
def load_from_db(self):
all_workers = get_workers()
- worker = [w for w in all_workers if w.pid == cint(self.name)][0]
- d = serialize_worker(worker)
+ workers = [w for w in all_workers if w.pid == cint(self.name)]
+ if not workers:
+ raise frappe.DoesNotExistError
+ d = serialize_worker(workers[0])
super(Document, self).__init__(d)
@@ -51,12 +53,15 @@ class RQWorker(Document):
def serialize_worker(worker: Worker) -> frappe._dict:
- queue = ", ".join(worker.queue_names())
+ queue_names = worker.queue_names()
+
+ queue = ", ".join(queue_names)
+ queue_types = ",".join(q.rsplit(":", 1)[1] for q in queue_names)
return frappe._dict(
name=worker.pid,
queue=queue,
- queue_type=queue.rsplit(":", 1)[1],
+ queue_type=queue_types,
worker_name=worker.name,
status=worker.get_state(),
pid=worker.pid,
diff --git a/frappe/core/doctype/rq_worker/test_rq_worker.py b/frappe/core/doctype/rq_worker/test_rq_worker.py
index 5a43270681..f07338d630 100644
--- a/frappe/core/doctype/rq_worker/test_rq_worker.py
+++ b/frappe/core/doctype/rq_worker/test_rq_worker.py
@@ -10,7 +10,7 @@ class TestRQWorker(FrappeTestCase):
def test_get_worker_list(self):
workers = RQWorker.get_list({})
self.assertGreaterEqual(len(workers), 1)
- self.assertTrue(any(w.queue_type == "short" for w in workers))
+ self.assertTrue(any("short" in w.queue_type for w in workers))
def test_worker_serialization(self):
workers = RQWorker.get_list({})
diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json
index 5fd5ef8163..57e2b1a1be 100644
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@@ -70,9 +70,6 @@
"hide_footer_in_auto_email_reports",
"attach_view_link",
"prepared_report_section",
- "enable_prepared_report_auto_deletion",
- "prepared_report_expiry_period",
- "column_break_64",
"max_auto_email_report_per_user",
"system_updates_section",
"disable_system_update_notification",
@@ -427,25 +424,11 @@
"label": "Send document Web View link in email"
},
{
- "default": "30",
- "depends_on": "enable_prepared_report_auto_deletion",
- "description": "System will auto-delete Prepared Reports permanently after these many days since creation",
- "fieldname": "prepared_report_expiry_period",
- "fieldtype": "Int",
- "label": "Prepared Report Expiry Period (Days)"
- },
- {
- "default": "1",
- "fieldname": "enable_prepared_report_auto_deletion",
- "fieldtype": "Check",
- "label": "Enable Auto-deletion of Prepared Reports"
- },
- {
- "collapsible": 1,
- "fieldname": "prepared_report_section",
- "fieldtype": "Section Break",
- "label": "Reports"
- },
+ "collapsible": 1,
+ "fieldname": "prepared_report_section",
+ "fieldtype": "Section Break",
+ "label": "Reports"
+ },
{
"default": "Frappe",
"description": "The application name will be used in the Login page.",
@@ -498,10 +481,6 @@
"fieldtype": "Check",
"label": "Allow Older Web View Links (Insecure)"
},
- {
- "fieldname": "column_break_64",
- "fieldtype": "Column Break"
- },
{
"default": "20",
"fieldname": "max_auto_email_report_per_user",
@@ -538,7 +517,7 @@
"icon": "fa fa-cog",
"issingle": 1,
"links": [],
- "modified": "2022-10-30 12:02:46.639170",
+ "modified": "2022-11-20 17:57:05.099512",
"modified_by": "Administrator",
"module": "Core",
"name": "System Settings",
diff --git a/frappe/database/database.py b/frappe/database/database.py
index dfcc9dfe58..5b775ccc10 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -2,6 +2,7 @@
# License: MIT. See LICENSE
import datetime
+import itertools
import json
import random
import re
@@ -9,6 +10,7 @@ import string
import traceback
from contextlib import contextmanager, suppress
from time import time
+from typing import Any, Iterable, Sequence
from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
from pypika.terms import Criterion, NullValue
@@ -1204,28 +1206,36 @@ class Database:
frappe.flags.touched_tables = set()
frappe.flags.touched_tables.update(tables)
- def bulk_insert(self, doctype, fields, values, ignore_duplicates=False, *, chunk_size=10_000):
+ def bulk_insert(
+ self,
+ doctype: str,
+ fields: list[str],
+ values: Iterable[Sequence[Any]],
+ ignore_duplicates=False,
+ *,
+ chunk_size=10_000,
+ ):
"""
Insert multiple records at a time
:param doctype: Doctype name
:param fields: list of fields
- :params values: list of list of values
+ :params values: iterable of values
"""
- values = list(values)
table = frappe.qb.DocType(doctype)
- for start_index in range(0, len(values), chunk_size):
- query = frappe.qb.into(table)
- if ignore_duplicates:
- # Pypika does not have same api for ignoring duplicates
- if self.db_type == "mariadb":
- query = query.ignore()
- elif self.db_type == "postgres":
- query = query.on_conflict().do_nothing()
+ query = frappe.qb.into(table).columns(fields)
- values_to_insert = values[start_index : start_index + chunk_size]
- query.columns(fields).insert(*values_to_insert).run()
+ if ignore_duplicates:
+ # Pypika does not have same api for ignoring duplicates
+ if frappe.conf.db_type == "mariadb":
+ query = query.ignore()
+ elif frappe.conf.db_type == "postgres":
+ query = query.on_conflict().do_nothing()
+
+ value_iterator = iter(values)
+ while value_chunk := tuple(itertools.islice(value_iterator, chunk_size)):
+ query.insert(*value_chunk).run()
def create_sequence(self, *args, **kwargs):
from frappe.database.sequence import create_sequence
diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py
index 322c355357..d8c806ffcf 100644
--- a/frappe/database/mariadb/database.py
+++ b/frappe/database/mariadb/database.py
@@ -119,7 +119,7 @@ class MariaDBConnectionUtil:
"use_unicode": True,
}
- if self.user != "root":
+ if self.user not in (frappe.flags.root_login, "root"):
conn_settings["database"] = self.user
if self.port:
diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py
index cb2b5508ce..5bb07aefe5 100644
--- a/frappe/desk/form/linked_with.py
+++ b/frappe/desk/form/linked_with.py
@@ -403,7 +403,6 @@ def get_exempted_doctypes():
return auto_cancel_exempt_doctypes
-@frappe.whitelist()
def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> dict[str, list]:
if isinstance(linkinfo, str):
# additional fields are added in linkinfo
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index d0bc63f858..a1e8a9368f 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -10,19 +10,12 @@ import frappe
import frappe.desk.reportview
from frappe import _
from frappe.core.utils import ljust_list
+from frappe.desk.reportview import clean_params, parse_json
from frappe.model.utils import render_include
from frappe.modules import get_module_path, scrub
from frappe.monitor import add_data_to_monitor
from frappe.permissions import get_role_permissions
-from frappe.utils import (
- cint,
- cstr,
- flt,
- format_duration,
- get_html_format,
- get_url_to_form,
- gzip_decompress,
-)
+from frappe.utils import cint, cstr, flt, format_duration, get_html_format
def get_report_doc(report_name):
@@ -144,35 +137,6 @@ def normalize_result(result, columns):
return data
-@frappe.whitelist()
-def background_enqueue_run(report_name, filters=None, user=None):
- """run reports in background"""
- if not user:
- user = frappe.session.user
- report = get_report_doc(report_name)
- track_instance = frappe.get_doc(
- {
- "doctype": "Prepared Report",
- "report_name": report_name,
- # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
- # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
- "filters": json.dumps(json.loads(filters)),
- "ref_report_doctype": report_name,
- "report_type": report.report_type,
- "query": report.query,
- "module": report.module,
- }
- )
- track_instance.insert(ignore_permissions=True)
- frappe.db.commit()
- track_instance.enqueue_report()
-
- return {
- "name": track_instance.name,
- "redirect_url": get_url_to_form("Prepared Report", track_instance.name),
- }
-
-
@frappe.whitelist()
def get_script(report_name):
report = get_report_doc(report_name)
@@ -230,18 +194,12 @@ def run(
result = None
- if (
- report.prepared_report
- and not report.disable_prepared_report
- and not ignore_prepared_report
- and not custom_columns
- ):
+ if report.prepared_report and not ignore_prepared_report and not custom_columns:
if filters:
if isinstance(filters, str):
filters = json.loads(filters)
- dn = filters.get("prepared_report_name")
- filters.pop("prepared_report_name", None)
+ dn = filters.pop("prepared_report_name", None)
else:
dn = ""
result = get_prepared_report_result(report, filters, dn, user)
@@ -271,103 +229,88 @@ def add_custom_column_data(custom_columns, result):
def get_prepared_report_result(report, filters, dn="", user=None):
- latest_report_data = {}
- doc = None
- if dn:
- # Get specified dn
- doc = frappe.get_doc("Prepared Report", dn)
- else:
- # Only look for completed prepared reports with given filters.
- doc_list = frappe.get_all(
- "Prepared Report",
- filters={
- "status": "Completed",
- "filters": json.dumps(filters),
- "owner": user,
- "report_name": report.get("custom_report") or report.get("report_name"),
- },
- order_by="creation desc",
+ from frappe.core.doctype.prepared_report.prepared_report import get_completed_prepared_report
+
+ def get_report_data(doc, data):
+ # backwards compatibility - prepared report used to have a columns field,
+ # we now directly fetch it from the result file
+ if doc.get("columns") or isinstance(data, list):
+ columns = (doc.get("columns") and json.loads(doc.columns)) or data[0]
+ data = {"result": data}
+ else:
+ columns = data.get("columns")
+
+ for column in columns:
+ if isinstance(column, dict) and column.get("label"):
+ column["label"] = _(column["label"])
+
+ return data | {"columns": columns}
+
+ report_data = {}
+ if not dn:
+ dn = get_completed_prepared_report(
+ filters, user, report.get("custom_report") or report.get("report_name")
)
- if doc_list:
- # Get latest
- doc = frappe.get_doc("Prepared Report", doc_list[0])
-
+ doc = frappe.get_doc("Prepared Report", dn) if dn else None
if doc:
try:
- # Prepared Report data is stored in a GZip compressed JSON file
- attached_file_name = frappe.db.get_value(
- "File",
- {"attached_to_doctype": doc.doctype, "attached_to_name": doc.name},
- "name",
- )
- attached_file = frappe.get_doc("File", attached_file_name)
- compressed_content = attached_file.get_content()
- uncompressed_content = gzip_decompress(compressed_content)
- data = json.loads(uncompressed_content.decode("utf-8"))
- if data:
- columns = json.loads(doc.columns) if doc.columns else data[0]
-
- for column in columns:
- if isinstance(column, dict) and column.get("label"):
- column["label"] = _(column["label"])
-
- latest_report_data = {"columns": columns, "result": data}
+ if data := json.loads(doc.get_prepared_data().decode("utf-8")):
+ report_data = get_report_data(doc, data)
except Exception:
- doc.log_error("Prepared report failed")
- frappe.delete_doc("Prepared Report", doc.name)
- frappe.db.commit()
+ doc.log_error("Prepared report render failed")
+ frappe.msgprint(_("Prepared report render failed"))
doc = None
- latest_report_data.update({"prepared_report": True, "doc": doc})
-
- return latest_report_data
+ return report_data | {"prepared_report": True, "doc": doc}
@frappe.whitelist()
def export_query():
"""export from query reports"""
- data = frappe._dict(frappe.local.form_dict)
- data.pop("cmd", None)
- data.pop("csrf_token", None)
+ from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
- if isinstance(data.get("filters"), str):
- filters = json.loads(data["filters"])
+ form_params = frappe._dict(frappe.local.form_dict)
+ csv_params = pop_csv_params(form_params)
+ clean_params(form_params)
+ parse_json(form_params)
- if data.get("report_name"):
- report_name = data["report_name"]
- frappe.permissions.can_export(
- frappe.get_cached_value("Report", report_name, "ref_doctype"),
- raise_exception=True,
- )
+ report_name = form_params.report_name
+ frappe.permissions.can_export(
+ frappe.get_cached_value("Report", report_name, "ref_doctype"),
+ raise_exception=True,
+ )
- file_format_type = data.get("file_format_type")
- custom_columns = frappe.parse_json(data.get("custom_columns", "[]"))
- include_indentation = data.get("include_indentation")
- visible_idx = data.get("visible_idx")
+ file_format_type = form_params.file_format_type
+ custom_columns = frappe.parse_json(form_params.custom_columns or "[]")
+ include_indentation = form_params.include_indentation
+ visible_idx = form_params.visible_idx
if isinstance(visible_idx, str):
visible_idx = json.loads(visible_idx)
- if file_format_type == "Excel":
- data = run(report_name, filters, custom_columns=custom_columns)
- data = frappe._dict(data)
- if not data.columns:
- frappe.respond_as_web_page(
- _("No data to export"),
- _("You can try changing the filters of your report."),
- )
- return
+ data = run(report_name, form_params.filters, custom_columns=custom_columns)
+ data = frappe._dict(data)
+ if not data.columns:
+ frappe.respond_as_web_page(
+ _("No data to export"),
+ _("You can try changing the filters of your report."),
+ )
+ return
+ format_duration_fields(data)
+ xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
+
+ if file_format_type == "CSV":
+ content = get_csv_bytes(xlsx_data, csv_params)
+ file_extension = "csv"
+ elif file_format_type == "Excel":
from frappe.utils.xlsxutils import make_xlsx
- format_duration_fields(data)
- xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
- xlsx_file = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths)
+ file_extension = "xlsx"
+ content = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths).getvalue()
- frappe.response["filename"] = report_name + ".xlsx"
- frappe.response["filecontent"] = xlsx_file.getvalue()
- frappe.response["type"] = "binary"
+ provide_binary_file(report_name, file_extension, content)
def format_duration_fields(data: frappe._dict) -> None:
diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py
index b24ab21455..5450d77377 100644
--- a/frappe/desk/reportview.py
+++ b/frappe/desk/reportview.py
@@ -4,7 +4,6 @@
"""build query for doclistview and return results"""
import json
-from io import StringIO
import frappe
import frappe.permissions
@@ -14,7 +13,7 @@ from frappe.model import child_table_fields, default_fields, optional_fields
from frappe.model.base_document import get_controller
from frappe.model.db_query import DatabaseQuery
from frappe.model.utils import is_virtual_doctype
-from frappe.utils import add_user_info, cstr, format_duration
+from frappe.utils import add_user_info, format_duration
@frappe.whitelist()
@@ -339,30 +338,21 @@ def delete_report(name):
@frappe.read_only()
def export_query():
"""export from report builder"""
- title = frappe.form_dict.title
- frappe.form_dict.pop("title", None)
+ from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
form_params = get_form_params()
form_params["limit_page_length"] = None
form_params["as_list"] = True
- doctype = form_params.doctype
- add_totals_row = None
- file_format_type = form_params["file_format_type"]
- title = title or doctype
-
- del form_params["doctype"]
- del form_params["file_format_type"]
-
- if "add_totals_row" in form_params and form_params["add_totals_row"] == "1":
- add_totals_row = 1
- del form_params["add_totals_row"]
+ doctype = form_params.pop("doctype")
+ file_format_type = form_params.pop("file_format_type")
+ title = form_params.pop("title", doctype)
+ csv_params = pop_csv_params(form_params)
+ add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None
frappe.permissions.can_export(doctype, raise_exception=True)
- if "selected_items" in form_params:
- si = json.loads(frappe.form_dict.get("selected_items"))
- form_params["filters"] = {"name": ("in", si)}
- del form_params["selected_items"]
+ if selection := form_params.pop("selected_items", None):
+ form_params["filters"] = {"name": ("in", json.loads(selection))}
make_access_log(
doctype=doctype,
@@ -378,38 +368,24 @@ def export_query():
ret = append_totals_row(ret)
data = [[_("Sr")] + get_labels(db_query.fields, doctype)]
- for i, row in enumerate(ret):
- data.append([i + 1] + list(row))
-
+ data.extend([i + 1] + list(row) for i, row in enumerate(ret))
data = handle_duration_fieldtype_values(doctype, data, db_query.fields)
if file_format_type == "CSV":
-
- # convert to csv
- import csv
-
from frappe.utils.xlsxutils import handle_html
- f = StringIO()
- writer = csv.writer(f)
- for r in data:
- # encode only unicode type strings and not int, floats etc.
- writer.writerow([handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r])
-
- f.seek(0)
- frappe.response["result"] = cstr(f.read())
- frappe.response["type"] = "csv"
- frappe.response["doctype"] = title
-
+ file_extension = "csv"
+ content = get_csv_bytes(
+ [[handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r] for r in data],
+ csv_params,
+ )
elif file_format_type == "Excel":
-
from frappe.utils.xlsxutils import make_xlsx
- xlsx_file = make_xlsx(data, doctype)
+ file_extension = "xlsx"
+ content = make_xlsx(data, doctype).getvalue()
- frappe.response["filename"] = title + ".xlsx"
- frappe.response["filecontent"] = xlsx_file.getvalue()
- frappe.response["type"] = "binary"
+ provide_binary_file(title, file_extension, content)
def append_totals_row(data):
@@ -436,16 +412,12 @@ def get_labels(fields, doctype):
"""get column labels based on column names"""
labels = []
for key in fields:
- key = key.split(" as ")[0]
-
- if key.startswith(("count(", "sum(", "avg(")):
+ try:
+ parenttype, fieldname = parse_field(key)
+ except ValueError:
continue
- if "." in key:
- parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
- else:
- parenttype = doctype
- fieldname = fieldname.strip("`")
+ parenttype = parenttype or doctype
if parenttype == doctype and fieldname == "name":
label = _("ID", context="Label of name column in report")
@@ -464,17 +436,12 @@ def get_labels(fields, doctype):
def handle_duration_fieldtype_values(doctype, data, fields):
for field in fields:
- key = field.split(" as ")[0]
-
- if key.startswith(("count(", "sum(", "avg(")):
+ try:
+ parenttype, fieldname = parse_field(field)
+ except ValueError:
continue
- if "." in key:
- parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
- else:
- parenttype = doctype
- fieldname = field.strip("`")
-
+ parenttype = parenttype or doctype
df = frappe.get_meta(parenttype).get_field(fieldname)
if df and df.fieldtype == "Duration":
@@ -487,6 +454,19 @@ def handle_duration_fieldtype_values(doctype, data, fields):
return data
+def parse_field(field: str) -> tuple[str | None, str]:
+ """Parse a field into parenttype and fieldname."""
+ key = field.split(" as ")[0]
+
+ if key.startswith(("count(", "sum(", "avg(")):
+ raise ValueError
+
+ if "." in key:
+ return key.split(".")[0][4:-1], key.split(".")[1].strip("`")
+
+ return None, key.strip("`")
+
+
@frappe.whitelist()
def delete_items():
"""delete selected items"""
diff --git a/frappe/desk/utils.py b/frappe/desk/utils.py
index 72cba79963..428ed95c02 100644
--- a/frappe/desk/utils.py
+++ b/frappe/desk/utils.py
@@ -27,3 +27,34 @@ def validate_route_conflict(doctype, name):
def slug(name):
return name.lower().replace(" ", "-")
+
+
+def pop_csv_params(form_dict):
+ """Pop csv params from form_dict and return them as a dict."""
+ from csv import QUOTE_NONNUMERIC
+
+ from frappe.utils.data import cint, cstr
+
+ return {
+ "delimiter": cstr(form_dict.pop("csv_delimiter", ","))[0],
+ "quoting": cint(form_dict.pop("csv_quoting", QUOTE_NONNUMERIC)),
+ }
+
+
+def get_csv_bytes(data: list[list], csv_params: dict) -> bytes:
+ """Convert data to csv bytes."""
+ from csv import writer
+ from io import StringIO
+
+ file = StringIO()
+ csv_writer = writer(file, **csv_params)
+ csv_writer.writerows(data)
+
+ return file.getvalue().encode("utf-8")
+
+
+def provide_binary_file(filename: str, extension: str, content: bytes) -> None:
+ """Provide a binary file to the client."""
+ frappe.response["type"] = "binary"
+ frappe.response["filecontent"] = content
+ frappe.response["filename"] = f"{filename}.{extension}"
diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py
index 3c020eea39..56f7f6f5ea 100644
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@@ -26,6 +26,7 @@ from frappe.utils import (
cstr,
get_hook_method,
get_string_between,
+ get_url,
nowdate,
sbool,
split_emails,
@@ -293,15 +294,11 @@ class SendMailContext:
message = self.include_attachments(message)
return message
- def get_tracker_str(self):
- tracker_url_html = '
'
-
- message = ""
+ def get_tracker_str(self) -> str:
if frappe.conf.use_ssl and self.email_account_doc.track_email_status:
- message = quopri.encodestring(
- tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
- ).decode()
- return message
+ tracker_url_html = f'}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={self.queue_doc.communication})
'
+ return quopri.encodestring(tracker_url_html.encode()).decode()
+ return ""
def get_unsubscribe_str(self, recipient_email: str) -> str:
unsubscribe_url = ""
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 73327f0ab1..ce52048c04 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -220,7 +220,6 @@ scheduler_events = {
"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
- "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports",
"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",
"frappe.utils.subscription.enable_manage_subscription",
],
diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index e689f91ddd..1d156d0d1a 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -384,14 +384,21 @@ class DatabaseQuery:
_raise_exception()
for field in self.fields:
+ lower_field = field.lower().strip()
+
if SUB_QUERY_PATTERN.match(field):
- if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords):
- _raise_exception()
+ if lower_field[0] == "(":
+ subquery_token = lower_field[1:].lstrip().split(" ", 1)[0]
+ if subquery_token in blacklisted_keywords:
+ _raise_exception()
- if any(f"{keyword}(" in field.lower() for keyword in blacklisted_functions):
- _raise_exception()
+ function = lower_field.split("(", 1)[0].rstrip()
+ if function in blacklisted_functions:
+ frappe.throw(
+ _("Use of function {0} in field is restricted").format(function), exc=frappe.DataError
+ )
- if "@" in field.lower():
+ if "@" in lower_field:
# prevent access to global variables
_raise_exception()
@@ -407,7 +414,7 @@ class DatabaseQuery:
if STRICT_FIELD_PATTERN.match(field):
frappe.throw(_("Illegal SQL Query"))
- if STRICT_UNION_PATTERN.match(field.lower()):
+ if STRICT_UNION_PATTERN.match(lower_field):
frappe.throw(_("Illegal SQL Query"))
def extract_tables(self):
@@ -904,12 +911,16 @@ class DatabaseQuery:
if self.order_by:
args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}"
- def validate_order_by_and_group_by(self, parameters):
+ def validate_order_by_and_group_by(self, parameters: str):
"""Check order by, group by so that atleast one column is selected and does not have subquery"""
if not parameters:
return
+ blacklisted_sql_functions = {
+ "sleep",
+ }
_lower = parameters.lower()
+
if "select" in _lower and "from" in _lower:
frappe.throw(_("Cannot use sub-query in order by"))
@@ -917,13 +928,20 @@ class DatabaseQuery:
frappe.throw(_("Illegal SQL Query"))
for field in parameters.split(","):
- if "." in field and field.strip().startswith("`tab"):
- tbl = field.strip().split(".")[0]
+ field = field.strip()
+ function = field.split("(", 1)[0].rstrip().lower()
+ full_field_name = "." in field and field.startswith("`tab")
+
+ if full_field_name:
+ tbl = field.split(".", 1)[0]
if tbl not in self.tables:
if tbl.startswith("`"):
tbl = tbl[4:-1]
frappe.throw(_("Please select atleast 1 column from {0} to sort/group").format(tbl))
+ if function in blacklisted_sql_functions:
+ frappe.throw(_("Cannot use {0} in order/group by").format(field))
+
def add_limit(self):
if self.limit_page_length:
return f"limit {self.limit_page_length} offset {self.limit_start}"
diff --git a/frappe/model/document.py b/frappe/model/document.py
index f5f710a578..c1d4575ae7 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -3,6 +3,7 @@
import hashlib
import json
import time
+from typing import Any, Generator, Iterable
from werkzeug.exceptions import NotFound
@@ -1593,3 +1594,40 @@ def execute_action(__doctype, __name, __action, **kwargs):
doc.add_comment("Comment", _("Action Failed") + "
" + msg)
doc.notify_update()
+
+
+def bulk_insert(
+ doctype: str,
+ documents: Iterable["Document"],
+ ignore_duplicates: bool = False,
+ chunk_size=10_000,
+):
+ """Insert simple Documents objects to database in bulk.
+
+ Warning/Info:
+ - All documents are inserted without triggering ANY hooks.
+ - This function assumes you've done the due dilligence and inserts in similar fashion as db_insert
+ - Documents can be any iterable / generator containing Document objects
+ """
+
+ columns = frappe.get_meta(doctype).get_valid_columns()
+ values = _document_values_generator(documents, columns)
+
+ frappe.db.bulk_insert(
+ doctype, columns, values, ignore_duplicates=ignore_duplicates, chunk_size=chunk_size
+ )
+
+
+def _document_values_generator(
+ documents: Iterable["Document"],
+ columns: list[str],
+) -> Generator[tuple[Any], None, None]:
+ for doc in documents:
+ doc.creation = doc.modified = now()
+ doc.created_by = doc.modified_by = frappe.session.user
+ doc_values = doc.get_valid_dict(
+ convert_dates_to_str=True,
+ ignore_nulls=True,
+ ignore_virtual=True,
+ )
+ yield tuple(doc_values.get(col) for col in columns)
diff --git a/frappe/model/meta.py b/frappe/model/meta.py
index 5a9c2d906d..ce30011d02 100644
--- a/frappe/model/meta.py
+++ b/frappe/model/meta.py
@@ -230,7 +230,7 @@ class Meta(Document):
return fields
- def get_valid_columns(self):
+ def get_valid_columns(self) -> list[str]:
if not hasattr(self, "_valid_columns"):
table_exists = frappe.db.table_exists(self.name)
if self.name in self.special_doctypes and table_exists:
diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py
index 09a91f21fd..0e124ee3aa 100644
--- a/frappe/modules/utils.py
+++ b/frappe/modules/utils.py
@@ -296,7 +296,7 @@ def make_boilerplate(
controller_body = indent(
dedent(
"""
- def db_insert(self):
+ def db_insert(self, *args, **kwargs):
pass
def load_from_db(self):
diff --git a/frappe/patches.txt b/frappe/patches.txt
index e8289a2790..9c298557c3 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -195,6 +195,8 @@ frappe.patches.v14_0.setup_likes_from_feedback
frappe.patches.v14_0.update_webforms
frappe.patches.v14_0.delete_payment_gateways
frappe.patches.v15_0.remove_event_streaming
+frappe.patches.v15_0.remove_prepared_report_settings_from_system_settings
+frappe.patches.v15_0.copy_disable_prepared_report_to_prepared_report
[post_model_sync]
frappe.patches.v14_0.drop_data_import_legacy
diff --git a/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py b/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py
new file mode 100644
index 0000000000..f1e4afb9a4
--- /dev/null
+++ b/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py
@@ -0,0 +1,6 @@
+import frappe
+
+
+def execute():
+ table = frappe.qb.DocType("Report")
+ frappe.qb.update(table).set(table.prepared_report, 0).where(table.disable_prepared_report == 1)
diff --git a/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py b/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py
new file mode 100644
index 0000000000..8c0ec4ca70
--- /dev/null
+++ b/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py
@@ -0,0 +1,15 @@
+import frappe
+from frappe.utils import cint
+
+
+def execute():
+ expiry_period = (
+ cint(frappe.db.get_single_value("System Settings", "prepared_report_expiry_period")) or 30
+ )
+ frappe.get_single("Log Settings").register_doctype("Prepared Report", expiry_period)
+
+ singles = frappe.qb.DocType("Singles")
+ frappe.qb.from_(singles).delete().where(
+ (singles.doctype == "System Settings")
+ & (singles.field.isin(["enable_prepared_report_auto_deletion", "prepared_report_expiry_period"]))
+ ).run()
diff --git a/frappe/public/js/frappe/form/controls/base_input.js b/frappe/public/js/frappe/form/controls/base_input.js
index 62f1c51f67..43fb4f54dc 100644
--- a/frappe/public/js/frappe/form/controls/base_input.js
+++ b/frappe/public/js/frappe/form/controls/base_input.js
@@ -83,7 +83,9 @@ frappe.ui.form.ControlInput = class ControlInput extends frappe.ui.form.Control
me.value = me.doc[me.df.fieldname] || "";
}
- if (me.can_write()) {
+ let is_fetch_from_read_only = me.df.fetch_from && !me.df.fetch_if_empty;
+
+ if (me.can_write() && !is_fetch_from_read_only) {
me.disp_area && $(me.disp_area).toggle(false);
$(me.input_area).toggle(true);
me.$input && me.$input.prop("disabled", false);
@@ -101,6 +103,16 @@ frappe.ui.form.ControlInput = class ControlInput extends frappe.ui.form.Control
}
}
me.$input && me.$input.prop("disabled", true);
+
+ if (is_fetch_from_read_only) {
+ $(me.disp_area).attr(
+ "title",
+ __(
+ "This value is fetched from {0}'s {1} field",
+ me.df.fetch_from.split(".")
+ )
+ );
+ }
}
me.set_description();
diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js
index d0c352d784..3d81744f59 100644
--- a/frappe/public/js/frappe/form/grid.js
+++ b/frappe/public/js/frappe/form/grid.js
@@ -61,57 +61,56 @@ export default class Grid {
make() {
let template = `
-
-
-
-
-
- ${__("No Data")}
+