-
-
-
-
-
- 
- ${__("No Data")}
+
- ${__("No Data")}
+
+
+
+
+
+
-
+
-
+
+
+
+
+
+ ${__("No Data")}
+
+ ${__("No Data")}
+
-
-
-
Date: Fri, 25 Nov 2022 09:08:30 +0000
Subject: [PATCH 27/72] chore: replace use of `cur_frm` with `this.frm`
(#19005)
---
frappe/public/js/frappe/form/linked_with.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/frappe/public/js/frappe/form/linked_with.js b/frappe/public/js/frappe/form/linked_with.js
index 4fc0a1a443..99b265851e 100644
--- a/frappe/public/js/frappe/form/linked_with.js
+++ b/frappe/public/js/frappe/form/linked_with.js
@@ -26,8 +26,8 @@ frappe.ui.form.LinkedWith = class LinkedWith {
this.dialog.on_page_show = () => {
frappe
.xcall("frappe.desk.form.linked_with.get", {
- doctype: cur_frm.doctype,
- docname: cur_frm.docname,
+ doctype: this.frm.doctype,
+ docname: this.frm.docname,
})
.then((r) => {
this.frm.__linked_docs = r;
From a4db4550e9974cfd00c669917aaaee13544d048b Mon Sep 17 00:00:00 2001
From: Marica
Date: Fri, 25 Nov 2022 20:55:16 +0530
Subject: [PATCH 28/72] fix: Avoid `update_order` twice on Kanban load (#19011)
* fix: Avoid `update_order` twice on kanban load
- The older logic called `kanban.update()` on a refresh (not on load) and returned from `render()`
- Due to a recent PR, it calls `kanban.update()` AND builds a new board on load, which invokes `update_order` twice consecutively and causes a timestamp error to be thrown
* chore: Remove redundant condition
Co-authored-by: Ritwik Puri
---
frappe/public/js/frappe/views/kanban/kanban_view.js | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/frappe/public/js/frappe/views/kanban/kanban_view.js b/frappe/public/js/frappe/views/kanban/kanban_view.js
index 7eccd7ebf8..17339258ab 100644
--- a/frappe/public/js/frappe/views/kanban/kanban_view.js
+++ b/frappe/public/js/frappe/views/kanban/kanban_view.js
@@ -175,9 +175,7 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView {
cur_list: this,
user_settings: this.view_user_settings,
});
- }
-
- if (this.kanban && board_name === this.kanban.board_name) {
+ } else if (board_name === this.kanban.board_name) {
this.kanban.update(this.data);
}
}
From 263055e247364fd683e7fa99e5aa0a869ee6c738 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Sat, 26 Nov 2022 12:45:57 +0530
Subject: [PATCH 29/72] fix: site creation using non-root users (#19014)
---
frappe/database/mariadb/database.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py
index 322c355357..d8c806ffcf 100644
--- a/frappe/database/mariadb/database.py
+++ b/frappe/database/mariadb/database.py
@@ -119,7 +119,7 @@ class MariaDBConnectionUtil:
"use_unicode": True,
}
- if self.user != "root":
+ if self.user not in (frappe.flags.root_login, "root"):
conn_settings["database"] = self.user
if self.port:
From d70e5e73e293daf9373a21c9b22b9edd61317682 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Sat, 26 Nov 2022 20:00:58 +0100
Subject: [PATCH 30/72] refactor: extract common methods
---
frappe/desk/query_report.py | 24 +++++---------------
frappe/desk/reportview.py | 44 +++++++++++++------------------------
frappe/desk/utils.py | 31 ++++++++++++++++++++++++++
3 files changed, 52 insertions(+), 47 deletions(-)
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index f83cc2d8ae..8b08352a04 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -5,7 +5,6 @@ import datetime
import json
import os
from datetime import timedelta
-from io import StringIO
import frappe
import frappe.desk.reportview
@@ -329,7 +328,10 @@ def get_prepared_report_result(report, filters, dn="", user=None):
@frappe.whitelist()
def export_query():
"""export from query reports"""
+ from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
+
form_params = frappe._dict(frappe.local.form_dict)
+ csv_params = pop_csv_params(form_params)
clean_params(form_params)
parse_json(form_params)
@@ -360,29 +362,15 @@ def export_query():
xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
if file_format_type == "CSV":
- import csv
-
- file = StringIO()
- writer = csv.writer(
- file,
- quoting=csv.QUOTE_NONNUMERIC
- if form_params.csv_quoting is None
- else cint(form_params.csv_quoting),
- delimiter=cstr(form_params.csv_delimiter or ","),
- )
- writer.writerows(xlsx_data)
- content = file.getvalue().encode("utf-8")
+ content = get_csv_bytes(xlsx_data, csv_params)
file_extension = "csv"
elif file_format_type == "Excel":
from frappe.utils.xlsxutils import make_xlsx
- file = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths)
file_extension = "xlsx"
- content = file.getvalue()
+ content = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths).getvalue()
- frappe.response["filename"] = f"{report_name}.{file_extension}"
- frappe.response["filecontent"] = content
- frappe.response["type"] = "binary"
+ provide_binary_file(report_name, file_extension, content)
def format_duration_fields(data: frappe._dict) -> None:
diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py
index d46f5c125a..9adbeee290 100644
--- a/frappe/desk/reportview.py
+++ b/frappe/desk/reportview.py
@@ -4,7 +4,6 @@
"""build query for doclistview and return results"""
import json
-from io import StringIO
import frappe
import frappe.permissions
@@ -14,7 +13,7 @@ from frappe.model import child_table_fields, default_fields, optional_fields
from frappe.model.base_document import get_controller
from frappe.model.db_query import DatabaseQuery
from frappe.model.utils import is_virtual_doctype
-from frappe.utils import add_user_info, cint, cstr, format_duration
+from frappe.utils import add_user_info, format_duration
@frappe.whitelist()
@@ -339,6 +338,8 @@ def delete_report(name):
@frappe.read_only()
def export_query():
"""export from report builder"""
+ from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
+
title = frappe.form_dict.title
frappe.form_dict.pop("title", None)
@@ -349,11 +350,7 @@ def export_query():
add_totals_row = None
file_format_type = form_params["file_format_type"]
title = title or doctype
- if file_format_type == "CSV":
- csv_delimiter = cstr(form_params.get("csv_delimiter", ","))
- csv_quoting = cint(form_params.get("csv_quoting", 2))
- del form_params["csv_delimiter"]
- del form_params["csv_quoting"]
+ csv_params = pop_csv_params(form_params)
del form_params["doctype"]
del form_params["file_format_type"]
@@ -382,37 +379,26 @@ def export_query():
if add_totals_row:
ret = append_totals_row(ret)
- data = [[_("Sr")] + get_labels(db_query.fields, doctype)]
- for i, row in enumerate(ret):
- data.append([i + 1] + list(row))
-
+ data = [[_("Sr")] + get_labels(db_query.fields, doctype)].extend(
+ [i + 1] + list(row) for i, row in enumerate(ret)
+ )
data = handle_duration_fieldtype_values(doctype, data, db_query.fields)
if file_format_type == "CSV":
- import csv
-
from frappe.utils.xlsxutils import handle_html
- f = StringIO()
- writer = csv.writer(f, quoting=csv_quoting, delimiter=csv_delimiter)
- for r in data:
- # encode only unicode type strings and not int, floats etc.
- writer.writerow([handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r])
-
- f.seek(0)
- frappe.response["result"] = cstr(f.read())
- frappe.response["type"] = "csv"
- frappe.response["doctype"] = title
-
+ file_extension = "csv"
+ content = get_csv_bytes(
+ [[handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r] for r in data],
+ csv_params,
+ )
elif file_format_type == "Excel":
-
from frappe.utils.xlsxutils import make_xlsx
- xlsx_file = make_xlsx(data, doctype)
+ file_extension = "xlsx"
+ content = make_xlsx(data, doctype).getvalue()
- frappe.response["filename"] = title + ".xlsx"
- frappe.response["filecontent"] = xlsx_file.getvalue()
- frappe.response["type"] = "binary"
+ provide_binary_file(title, file_extension, content)
def append_totals_row(data):
diff --git a/frappe/desk/utils.py b/frappe/desk/utils.py
index 72cba79963..428ed95c02 100644
--- a/frappe/desk/utils.py
+++ b/frappe/desk/utils.py
@@ -27,3 +27,34 @@ def validate_route_conflict(doctype, name):
def slug(name):
return name.lower().replace(" ", "-")
+
+
+def pop_csv_params(form_dict):
+ """Pop csv params from form_dict and return them as a dict."""
+ from csv import QUOTE_NONNUMERIC
+
+ from frappe.utils.data import cint, cstr
+
+ return {
+ "delimiter": cstr(form_dict.pop("csv_delimiter", ","))[0],
+ "quoting": cint(form_dict.pop("csv_quoting", QUOTE_NONNUMERIC)),
+ }
+
+
+def get_csv_bytes(data: list[list], csv_params: dict) -> bytes:
+ """Convert data to csv bytes."""
+ from csv import writer
+ from io import StringIO
+
+ file = StringIO()
+ csv_writer = writer(file, **csv_params)
+ csv_writer.writerows(data)
+
+ return file.getvalue().encode("utf-8")
+
+
+def provide_binary_file(filename: str, extension: str, content: bytes) -> None:
+ """Provide a binary file to the client."""
+ frappe.response["type"] = "binary"
+ frappe.response["filecontent"] = content
+ frappe.response["filename"] = f"{filename}.{extension}"
From 839de22d956fbde00eb19760b7eece1007a7e166 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Sat, 26 Nov 2022 20:01:48 +0100
Subject: [PATCH 31/72] feat: add test for exporting reportview as CSV
---
frappe/tests/test_reportview.py | 34 +++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
create mode 100644 frappe/tests/test_reportview.py
diff --git a/frappe/tests/test_reportview.py b/frappe/tests/test_reportview.py
new file mode 100644
index 0000000000..ba3657aacd
--- /dev/null
+++ b/frappe/tests/test_reportview.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
+# License: MIT. See LICENSE
+
+import frappe
+from frappe.desk.reportview import export_query
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestReportview(FrappeTestCase):
+ def test_csv(self):
+ from csv import QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC, DictReader
+ from io import StringIO
+
+ frappe.local.form_dict = frappe._dict(
+ doctype="DocType",
+ title="Test Report",
+ file_format_type="CSV",
+ )
+
+ for delimiter in (",", ";", "\t", "|"):
+ for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC):
+ frappe.local.form_dict.update(
+ {
+ "csv_quoting": quoting,
+ "csv_delimiter": delimiter,
+ }
+ )
+ export_query()
+
+ self.assertTrue(frappe.response["filename"].endswith(".csv"))
+ self.assertEqual(frappe.response["type"], "binary")
+ with StringIO(frappe.response["filecontent"].decode("utf-8")) as result:
+ reader = DictReader(result, delimiter=delimiter, quoting=quoting)
+ reader.__next__()
From 433b285b92b5cd58db50b38532d5d4ff943ef311 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Sat, 26 Nov 2022 20:08:50 +0100
Subject: [PATCH 32/72] refactor: use provide_binary_file
---
frappe/core/doctype/data_export/exporter.py | 11 ++++-------
frappe/core/doctype/data_import/exporter.py | 12 ++----------
frappe/utils/xlsxutils.py | 8 +++-----
3 files changed, 9 insertions(+), 22 deletions(-)
diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py
index 611592531d..366245b5e8 100644
--- a/frappe/core/doctype/data_export/exporter.py
+++ b/frappe/core/doctype/data_export/exporter.py
@@ -432,23 +432,20 @@ class DataExporter:
row[_column_start_end.start + i + 1] = value
def build_response_as_excel(self):
+ from frappe.desk.utils import provide_binary_file
+ from frappe.utils.xlsxutils import make_xlsx
+
filename = frappe.generate_hash(length=10)
with open(filename, "wb") as f:
f.write(cstr(self.writer.getvalue()).encode("utf-8"))
f = open(filename)
reader = csv.reader(f)
-
- from frappe.utils.xlsxutils import make_xlsx
-
xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else "Data Export")
f.close()
os.remove(filename)
- # write out response as a xlsx type
- frappe.response["filename"] = self.doctype + ".xlsx"
- frappe.response["filecontent"] = xlsx_file.getvalue()
- frappe.response["type"] = "binary"
+ provide_binary_file(self.doctype, "xlsx", xlsx_file.getvalue())
def _append_name_column(self, dt=None):
self.append_field_column(
diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py
index 9c793767e3..1066b8bed9 100644
--- a/frappe/core/doctype/data_import/exporter.py
+++ b/frappe/core/doctype/data_import/exporter.py
@@ -1,8 +1,6 @@
# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
-import typing
-
import frappe
from frappe import _
from frappe.model import display_fieldtypes, no_value_fields
@@ -241,15 +239,9 @@ class Exporter:
def build_response(self):
if self.file_type == "CSV":
- self.build_csv_response()
+ build_csv_response(self.get_csv_array_for_export(), _(self.doctype))
elif self.file_type == "Excel":
- self.build_xlsx_response()
-
- def build_csv_response(self):
- build_csv_response(self.get_csv_array_for_export(), _(self.doctype))
-
- def build_xlsx_response(self):
- build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype))
+ build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype))
def group_children_data_by_parent(self, children_data: dict[str, list]):
return groupby_metric(children_data, key="parent")
diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py
index fc4ef33e88..d003b3a8a4 100644
--- a/frappe/utils/xlsxutils.py
+++ b/frappe/utils/xlsxutils.py
@@ -108,8 +108,6 @@ def read_xls_file_from_attached_file(content):
def build_xlsx_response(data, filename):
- xlsx_file = make_xlsx(data, filename)
- # write out response as a xlsx type
- frappe.response["filename"] = filename + ".xlsx"
- frappe.response["filecontent"] = xlsx_file.getvalue()
- frappe.response["type"] = "binary"
+ from frappe.desk.utils import provide_binary_file
+
+ provide_binary_file(filename, "xlsx", make_xlsx(data, filename).getvalue())
From d7139af6ccc291931f6050f383989a32ada1ca9a Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Sun, 27 Nov 2022 17:02:25 +0100
Subject: [PATCH 33/72] feat: hide csv settings in collapsible section
---
.../js/frappe/views/reports/report_utils.js | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js
index faf5c4af4c..d96461a297 100644
--- a/frappe/public/js/frappe/views/reports/report_utils.js
+++ b/frappe/public/js/frappe/views/reports/report_utils.js
@@ -169,6 +169,13 @@ frappe.report_utils = {
default: "Excel",
reqd: 1,
},
+ {
+ fieldtype: "Section Break",
+ fieldname: "csv_settings",
+ label: __("Settings", null, "Export report"),
+ collapsible: 1,
+ depends_on: "eval:doc.file_format=='CSV'",
+ },
{
fieldtype: "Data",
label: __("CSV Delimiter", null, "Export report"),
@@ -200,7 +207,14 @@ frappe.report_utils = {
];
if (extra_fields) {
- fields.push(...extra_fields);
+ fields.push(
+ {
+ fieldtype: "Section Break",
+ fieldname: "extra_fields",
+ collapsible: 0,
+ },
+ ...extra_fields
+ );
}
const dialog = new frappe.ui.Dialog({
From 1d254371214ee68c2b5a34089041cb5686bfdab1 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Sun, 27 Nov 2022 17:10:53 +0100
Subject: [PATCH 34/72] fix: duplicate translation of field labels
---
.../js/frappe/views/reports/report_utils.js | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js
index d96461a297..9713f8bb99 100644
--- a/frappe/public/js/frappe/views/reports/report_utils.js
+++ b/frappe/public/js/frappe/views/reports/report_utils.js
@@ -162,7 +162,7 @@ frappe.report_utils = {
get_export_dialog(report_name, extra_fields, callback) {
const fields = [
{
- label: __("Select File Format", null, "Export report"),
+ label: "File Format",
fieldname: "file_format",
fieldtype: "Select",
options: ["Excel", "CSV"],
@@ -172,13 +172,13 @@ frappe.report_utils = {
{
fieldtype: "Section Break",
fieldname: "csv_settings",
- label: __("Settings", null, "Export report"),
+ label: "Settings",
collapsible: 1,
depends_on: "eval:doc.file_format=='CSV'",
},
{
fieldtype: "Data",
- label: __("CSV Delimiter", null, "Export report"),
+ label: "CSV Delimiter",
fieldname: "csv_delimiter",
default: ",",
length: 1,
@@ -186,20 +186,20 @@ frappe.report_utils = {
},
{
fieldtype: "Select",
- label: __("CSV Quoting", null, "Export report"),
+ label: "CSV Quoting",
fieldname: "csv_quoting",
options: [
- { value: 0, label: __("Minimal", null, "Export report") },
- { value: 1, label: __("All", null, "Export report") },
- { value: 2, label: __("Non-numeric", null, "Export report") },
- { value: 3, label: __("None", null, "Export report") },
+ { value: 0, label: "Minimal" },
+ { value: 1, label: "All" },
+ { value: 2, label: "Non-numeric" },
+ { value: 3, label: "None" },
],
default: 2,
depends_on: "eval:doc.file_format=='CSV'",
},
{
fieldtype: "Small Text",
- label: __("CSV Preview", null, "Export report"),
+ label: "CSV Preview",
fieldname: "csv_preview",
read_only: 1,
depends_on: "eval:doc.file_format=='CSV'",
From b8b6db541afcb64c58171be9eb755062b15f964d Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Sun, 27 Nov 2022 17:13:22 +0100
Subject: [PATCH 35/72] feat: german translations for export dialog
---
frappe/translations/de.csv | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv
index b1000f3bee..6cd8133f94 100644
--- a/frappe/translations/de.csv
+++ b/frappe/translations/de.csv
@@ -158,7 +158,7 @@ No Data,Keine Daten,
No address added yet.,Noch keine Adresse hinzugefügt.,
No contacts added yet.,Noch keine Kontakte hinzugefügt.,
No items found.,Keine Elemente gefunden.,
-None,Keiner,
+None,Keine,
Not Permitted,Nicht zulässig,
Not active,Nicht aktiv,
Notes,Hinweise,
@@ -2202,7 +2202,6 @@ Select Columns,Spalten auswählen,
Select Document Type,Dokumenttyp auswählen,
Select Document Type or Role to start.,"Dokumententyp oder Rolle auswählen, um zu beginnen.",
Select Document Types to set which User Permissions are used to limit access.,"Dokumentenarten auswählen, um die Benutzerrechte, die den Zugriff einschränken, anzuwenden",
-Select File Format,Wählen Sie Dateiformat,
Select File Type,Dateityp auswählen,
Select Language...,Sprache auswählen...,
Select Languages,Sprachenauswahl,
@@ -4823,3 +4822,9 @@ Preview Chart,Vorschau erzeugen,
Please select X and Y fields,Bitte Felder für die X- und Y-Achse wählen,
Notification sent to,Benachrichtigung gesendet an,
Add to this activity by mailing to {0},"Senden Sie eine E-Mail an {0}, damit sie hier erscheint",
+File Format,Dateiformat,
+CSV Delimiter,Trennzeichen,
+CSV Quoting,Anführungszeichen,
+CSV Preview,Vorschau,
+Non-numeric,Nicht-numerische,
+Minimal,Minimal,
From 406f8941415ad25a334bbbd2d8b14a1dfb9a1604 Mon Sep 17 00:00:00 2001
From: Sabu Siyad
Date: Mon, 28 Nov 2022 11:09:15 +0530
Subject: [PATCH 36/72] feat: default value options: sort (#19019)
[skip ci]
Signed-off-by: Sabu Siyad
---
frappe/core/doctype/doctype/doctype.js | 2 ++
1 file changed, 2 insertions(+)
diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js
index 24c367b115..7324a92359 100644
--- a/frappe/core/doctype/doctype/doctype.js
+++ b/frappe/core/doctype/doctype/doctype.js
@@ -124,6 +124,7 @@ frappe.ui.form.on("DocField", {
let doctypes = frm.doc.fields
.filter((df) => df.fieldtype == "Link")
.filter((df) => df.options && df.fieldname != row.fieldname)
+ .sort((a, b) => a.options.localeCompare(b.options))
.map((df) => ({
label: `${df.options} (${df.fieldname})`,
value: df.fieldname,
@@ -151,6 +152,7 @@ frappe.ui.form.on("DocField", {
.get_docfields(link_doctype, null, {
fieldtype: ["not in", frappe.model.no_value_type],
})
+ .sort((a, b) => a.label.localeCompare(b.label))
.map((df) => ({
label: `${df.label} (${df.fieldtype})`,
value: df.fieldname,
From a534e7a66a5b514203002e1e417574d69fdb8cb0 Mon Sep 17 00:00:00 2001
From: Mohammad Hussain Nagaria
<34810212+NagariaHussain@users.noreply.github.com>
Date: Mon, 28 Nov 2022 11:14:24 +0530
Subject: [PATCH 37/72] fix: Pass *args and **kwargs in `db_insert` boilerplate
for Virtual DocType (#19018)
[skip ci]
---
frappe/modules/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py
index 09a91f21fd..0e124ee3aa 100644
--- a/frappe/modules/utils.py
+++ b/frappe/modules/utils.py
@@ -296,7 +296,7 @@ def make_boilerplate(
controller_body = indent(
dedent(
"""
- def db_insert(self):
+ def db_insert(self, *args, **kwargs):
pass
def load_from_db(self):
From 7504c1bc738291b17c715de8737bcaa09599625b Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Mon, 28 Nov 2022 11:31:24 +0530
Subject: [PATCH 38/72] refactor: simpler commit after_request
---
frappe/app.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/frappe/app.py b/frappe/app.py
index fc679aa44e..5bacf8d4ec 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -320,12 +320,13 @@ def handle_exception(e):
def after_request(rollback):
# if HTTP method would change server state, commit if necessary
- if frappe.db and (
- frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS
+ if (
+ frappe.db
+ and (frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS)
+ and frappe.db.transaction_writes
):
- if frappe.db.transaction_writes:
- frappe.db.commit()
- rollback = False
+ frappe.db.commit()
+ rollback = False
# update session
if getattr(frappe.local, "session_obj", None):
From 4cc367f6c1d9ada5c27101f9ef10f7c5fab66f76 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Mon, 28 Nov 2022 11:34:46 +0530
Subject: [PATCH 39/72] fix: apply rollback to all HTTP methods that can write
---
frappe/app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/frappe/app.py b/frappe/app.py
index 5bacf8d4ec..3c768c3d1d 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -77,7 +77,7 @@ def application(request: Request):
rollback = after_request(rollback)
finally:
- if request.method in ("POST", "PUT") and frappe.db and rollback:
+ if request.method in UNSAFE_HTTP_METHODS and frappe.db and rollback:
frappe.db.rollback()
frappe.rate_limiter.update()
From 4f162da728ac32a7831e641bc8a5545941ea7524 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Mon, 28 Nov 2022 11:39:22 +0530
Subject: [PATCH 40/72] fix: rollback if not commiting
---
frappe/app.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/frappe/app.py b/frappe/app.py
index 3c768c3d1d..2fe9991c4c 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -327,6 +327,9 @@ def after_request(rollback):
):
frappe.db.commit()
rollback = False
+ elif frappe.db:
+ frappe.db.rollback()
+ rollback = False
# update session
if getattr(frappe.local, "session_obj", None):
From 6062d812a16f5e6e7e2e63b9cc1be99585d105ab Mon Sep 17 00:00:00 2001
From: gavin
Date: Mon, 28 Nov 2022 12:42:29 +0530
Subject: [PATCH 41/72] fix(db_query): Disallow usage of certain functions in
*_by (#18981)
* fix(db_query): Disallow blacklisted functions in (order|group)_by
Changes:
- allow only functions that are not blacklisted in *_by clause:
currently just sleep
- perf improvemnts: lower, in, split, strip & other low hanging micro optimizations
Handle the following use cases:
- upper/lower case function usages
- spaces between function name and brackets
* test(db_query): Add tests for *_by checks
---
frappe/model/db_query.py | 17 ++++++++++++---
frappe/tests/test_db_query.py | 40 +++++++++++++++++++++++++++++++++++
2 files changed, 54 insertions(+), 3 deletions(-)
diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 4b9318ab03..1d156d0d1a 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -911,12 +911,16 @@ class DatabaseQuery:
if self.order_by:
args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}"
- def validate_order_by_and_group_by(self, parameters):
+ def validate_order_by_and_group_by(self, parameters: str):
"""Check order by, group by so that atleast one column is selected and does not have subquery"""
if not parameters:
return
+ blacklisted_sql_functions = {
+ "sleep",
+ }
_lower = parameters.lower()
+
if "select" in _lower and "from" in _lower:
frappe.throw(_("Cannot use sub-query in order by"))
@@ -924,13 +928,20 @@ class DatabaseQuery:
frappe.throw(_("Illegal SQL Query"))
for field in parameters.split(","):
- if "." in field and field.strip().startswith("`tab"):
- tbl = field.strip().split(".")[0]
+ field = field.strip()
+ function = field.split("(", 1)[0].rstrip().lower()
+ full_field_name = "." in field and field.startswith("`tab")
+
+ if full_field_name:
+ tbl = field.split(".", 1)[0]
if tbl not in self.tables:
if tbl.startswith("`"):
tbl = tbl[4:-1]
frappe.throw(_("Please select atleast 1 column from {0} to sort/group").format(tbl))
+ if function in blacklisted_sql_functions:
+ frappe.throw(_("Cannot use {0} in order/group by").format(field))
+
def add_limit(self):
if self.limit_page_length:
return f"limit {self.limit_page_length} offset {self.limit_start}"
diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py
index 52f18f5963..1964792ea8 100644
--- a/frappe/tests/test_db_query.py
+++ b/frappe/tests/test_db_query.py
@@ -550,6 +550,46 @@ class TestReportview(FrappeTestCase):
)
self.assertTrue("DefaultValue" in [d["name"] for d in out])
+ def test_order_by_group_by_sanitizer(self):
+ # order by with blacklisted function
+ with self.assertRaises(frappe.ValidationError):
+ DatabaseQuery("DocType").execute(
+ fields=["name"],
+ order_by="sleep (1) asc",
+ )
+
+ # group by with blacklisted function
+ with self.assertRaises(frappe.ValidationError):
+ DatabaseQuery("DocType").execute(
+ fields=["name"],
+ group_by="SLEEP(0)",
+ )
+
+ # sub query in order by
+ with self.assertRaises(frappe.ValidationError):
+ DatabaseQuery("DocType").execute(
+ fields=["name"],
+ order_by="(select rank from tabRankedDocTypes where tabRankedDocTypes.name = tabDocType.name) asc",
+ )
+
+ # validate allowed usage
+ DatabaseQuery("DocType").execute(
+ fields=["name"],
+ order_by="name asc",
+ )
+ DatabaseQuery("DocType").execute(
+ fields=["name"],
+ order_by="name asc",
+ group_by="name",
+ )
+
+ # check mariadb specific syntax
+ if frappe.db.db_type == "mariadb":
+ DatabaseQuery("DocType").execute(
+ fields=["name"],
+ order_by="timestamp(modified)",
+ )
+
def test_of_not_of_descendant_ancestors(self):
frappe.set_user("Administrator")
clear_user_permissions_for_doctype("Nested DocType")
From 7ec120d2baaaa0b3d211c73c73f61dbe6f3138e4 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Sat, 11 Jun 2022 21:58:34 +0530
Subject: [PATCH 42/72] refactor: support generators in db.bulk_insert
When you want to insert a huge amount of docs in DB, using bulk insert
currently requries creating a huge number of objects, this is
ineffecient from memory perspective and also delays failures if any till
last step in execution.
---
frappe/database/database.py | 36 +++++++++++++++++++++++-------------
1 file changed, 23 insertions(+), 13 deletions(-)
diff --git a/frappe/database/database.py b/frappe/database/database.py
index dfcc9dfe58..a6003b7c03 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -3,12 +3,14 @@
import datetime
import json
+import itertools
import random
import re
import string
import traceback
from contextlib import contextmanager, suppress
from time import time
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
from pypika.terms import Criterion, NullValue
@@ -1204,28 +1206,36 @@ class Database:
frappe.flags.touched_tables = set()
frappe.flags.touched_tables.update(tables)
- def bulk_insert(self, doctype, fields, values, ignore_duplicates=False, *, chunk_size=10_000):
+ def bulk_insert(
+ self,
+ doctype: str,
+ fields: List[str],
+ values: Iterable[Sequence[Any]],
+ ignore_duplicates=False,
+ *,
+ chunk_size=10_000,
+ ):
"""
Insert multiple records at a time
:param doctype: Doctype name
:param fields: list of fields
- :params values: list of list of values
+ :params values: iterable of values
"""
- values = list(values)
table = frappe.qb.DocType(doctype)
- for start_index in range(0, len(values), chunk_size):
- query = frappe.qb.into(table)
- if ignore_duplicates:
- # Pypika does not have same api for ignoring duplicates
- if self.db_type == "mariadb":
- query = query.ignore()
- elif self.db_type == "postgres":
- query = query.on_conflict().do_nothing()
+ query = frappe.qb.into(table).columns(fields)
- values_to_insert = values[start_index : start_index + chunk_size]
- query.columns(fields).insert(*values_to_insert).run()
+ if ignore_duplicates:
+ # Pypika does not have same api for ignoring duplicates
+ if frappe.conf.db_type == "mariadb":
+ query = query.ignore()
+ elif frappe.conf.db_type == "postgres":
+ query = query.on_conflict().do_nothing()
+
+ value_iterator = iter(values)
+ while value_chunk := tuple(itertools.islice(value_iterator, chunk_size)):
+ query.insert(*value_chunk).run()
def create_sequence(self, *args, **kwargs):
from frappe.database.sequence import create_sequence
From a6af1ed542e39ce5d146ae34daf4fabacb308a68 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Sat, 11 Jun 2022 22:00:53 +0530
Subject: [PATCH 43/72] feat: bulk insert simple `Document` objects
---
frappe/database/database.py | 6 +++---
frappe/model/document.py | 38 +++++++++++++++++++++++++++++++++++
frappe/model/meta.py | 2 +-
frappe/tests/test_document.py | 18 +++++++++++++++++
4 files changed, 60 insertions(+), 4 deletions(-)
diff --git a/frappe/database/database.py b/frappe/database/database.py
index a6003b7c03..5b775ccc10 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -2,15 +2,15 @@
# License: MIT. See LICENSE
import datetime
-import json
import itertools
+import json
import random
import re
import string
import traceback
from contextlib import contextmanager, suppress
from time import time
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+from typing import Any, Iterable, Sequence
from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
from pypika.terms import Criterion, NullValue
@@ -1209,7 +1209,7 @@ class Database:
def bulk_insert(
self,
doctype: str,
- fields: List[str],
+ fields: list[str],
values: Iterable[Sequence[Any]],
ignore_duplicates=False,
*,
diff --git a/frappe/model/document.py b/frappe/model/document.py
index f5f710a578..c1d4575ae7 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -3,6 +3,7 @@
import hashlib
import json
import time
+from typing import Any, Generator, Iterable
from werkzeug.exceptions import NotFound
@@ -1593,3 +1594,40 @@ def execute_action(__doctype, __name, __action, **kwargs):
doc.add_comment("Comment", _("Action Failed") + "
" + msg) doc.notify_update() + + +def bulk_insert( + doctype: str, + documents: Iterable["Document"], + ignore_duplicates: bool = False, + chunk_size=10_000, +): + """Insert simple Documents objects to database in bulk. + + Warning/Info: + - All documents are inserted without triggering ANY hooks. + - This function assumes you've done the due dilligence and inserts in similar fashion as db_insert + - Documents can be any iterable / generator containing Document objects + """ + + columns = frappe.get_meta(doctype).get_valid_columns() + values = _document_values_generator(documents, columns) + + frappe.db.bulk_insert( + doctype, columns, values, ignore_duplicates=ignore_duplicates, chunk_size=chunk_size + ) + + +def _document_values_generator( + documents: Iterable["Document"], + columns: list[str], +) -> Generator[tuple[Any], None, None]: + for doc in documents: + doc.creation = doc.modified = now() + doc.created_by = doc.modified_by = frappe.session.user + doc_values = doc.get_valid_dict( + convert_dates_to_str=True, + ignore_nulls=True, + ignore_virtual=True, + ) + yield tuple(doc_values.get(col) for col in columns) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 5a9c2d906d..ce30011d02 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -230,7 +230,7 @@ class Meta(Document): return fields - def get_valid_columns(self): + def get_valid_columns(self) -> list[str]: if not hasattr(self, "_valid_columns"): table_exists = frappe.db.table_exists(self.name) if self.name in self.special_doctypes and table_exists: diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 3833e911a7..4ad2d85a19 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -469,3 +469,21 @@ class TestDocumentWebView(FrappeTestCase): # Logged-in user can access the page without key self.assertEqual(self.get(url_without_key, "Administrator").status, "200 OK") + + def test_bulk_inserts(self): + from frappe.model.document import bulk_insert + + doctype = "ToDo" + sent_todo = set() + + def doc_generator(): + for i in range(690): + doc = frappe.new_doc(doctype) + doc.name = doc.description = frappe.generate_hash() + sent_todo.add(doc.name) + yield doc + + bulk_insert(doctype, doc_generator(), chunk_size=100) + + all_todos = set(frappe.get_all("ToDo", pluck="name")) + self.assertEqual(sent_todo - all_todos, set(), "All docs should be inserted") From 5008bc0b093b162e43f51cf1bdb130dc3704d7cf Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:15:30 +0100 Subject: [PATCH 44/72] fix: fieldname referenced before assignment --- frappe/desk/reportview.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 9adbeee290..3e5ee0bc92 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -425,16 +425,12 @@ def get_labels(fields, doctype): """get column labels based on column names""" labels = [] for key in fields: - key = key.split(" as ")[0] - - if key.startswith(("count(", "sum(", "avg(")): + try: + parenttype, fieldname = parse_field(key) + except ValueError: continue - if "." in key: - parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") - else: - parenttype = doctype - fieldname = fieldname.strip("`") + parenttype = parenttype or doctype if parenttype == doctype and fieldname == "name": label = _("ID", context="Label of name column in report") @@ -453,17 +449,12 @@ def get_labels(fields, doctype): def handle_duration_fieldtype_values(doctype, data, fields): for field in fields: - key = field.split(" as ")[0] - - if key.startswith(("count(", "sum(", "avg(")): + try: + parenttype, fieldname = parse_field(field) + except ValueError: continue - if "." in key: - parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") - else: - parenttype = doctype - fieldname = field.strip("`") - + parenttype = parenttype or doctype df = frappe.get_meta(parenttype).get_field(fieldname) if df and df.fieldtype == "Duration": @@ -476,6 +467,19 @@ def handle_duration_fieldtype_values(doctype, data, fields): return data +def parse_field(field: str) -> tuple[str | None, str]: + """Parse a field into parenttype and fieldname.""" + key = field.split(" as ")[0] + + if key.startswith(("count(", "sum(", "avg(")): + raise ValueError + + if "." in key: + return key.split(".")[0][4:-1], key.split(".")[1].strip("`") + + return None, key.strip("`") + + @frappe.whitelist() def delete_items(): """delete selected items""" From ee18984e1f389aecd44db5c21f512787b2141cc1 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:15:57 +0100 Subject: [PATCH 45/72] refactor: export_query --- frappe/desk/reportview.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 3e5ee0bc92..8428f13af5 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -340,31 +340,19 @@ def export_query(): """export from report builder""" from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file - title = frappe.form_dict.title - frappe.form_dict.pop("title", None) - form_params = get_form_params() form_params["limit_page_length"] = None form_params["as_list"] = True - doctype = form_params.doctype - add_totals_row = None - file_format_type = form_params["file_format_type"] - title = title or doctype + doctype = form_params.pop("doctype") + file_format_type = form_params.pop("file_format_type") + title = frappe.form_dict.pop("title", doctype) csv_params = pop_csv_params(form_params) - - del form_params["doctype"] - del form_params["file_format_type"] - - if "add_totals_row" in form_params and form_params["add_totals_row"] == "1": - add_totals_row = 1 - del form_params["add_totals_row"] + add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None frappe.permissions.can_export(doctype, raise_exception=True) - if "selected_items" in form_params: - si = json.loads(frappe.form_dict.get("selected_items")) - form_params["filters"] = {"name": ("in", si)} - del form_params["selected_items"] + if selection := form_params.pop("selected_items", None): + form_params["filters"] = {"name": ("in", json.loads(selection))} make_access_log( doctype=doctype, @@ -379,9 +367,8 @@ def export_query(): if add_totals_row: ret = append_totals_row(ret) - data = [[_("Sr")] + get_labels(db_query.fields, doctype)].extend( - [i + 1] + list(row) for i, row in enumerate(ret) - ) + data = [[_("Sr")] + get_labels(db_query.fields, doctype)] + data.extend([i + 1] + list(row) for i, row in enumerate(ret)) data = handle_duration_fieldtype_values(doctype, data, db_query.fields) if file_format_type == "CSV": From 2c0827dda95f14c620b80dd20030bba557afe990 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:16:18 +0100 Subject: [PATCH 46/72] fix: test reportview --- frappe/tests/test_reportview.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_reportview.py b/frappe/tests/test_reportview.py index ba3657aacd..4a24514acf 100644 --- a/frappe/tests/test_reportview.py +++ b/frappe/tests/test_reportview.py @@ -13,22 +13,22 @@ class TestReportview(FrappeTestCase): frappe.local.form_dict = frappe._dict( doctype="DocType", - title="Test Report", file_format_type="CSV", + fields=("name", "module", "issingle"), + filters={"issingle": 1, "module": "Core"}, ) for delimiter in (",", ";", "\t", "|"): + frappe.local.form_dict.csv_delimiter = delimiter for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC): - frappe.local.form_dict.update( - { - "csv_quoting": quoting, - "csv_delimiter": delimiter, - } - ) + frappe.local.form_dict.csv_quoting = quoting + export_query() self.assertTrue(frappe.response["filename"].endswith(".csv")) self.assertEqual(frappe.response["type"], "binary") with StringIO(frappe.response["filecontent"].decode("utf-8")) as result: reader = DictReader(result, delimiter=delimiter, quoting=quoting) - reader.__next__() + for row in reader: + self.assertEqual(int(row["Is Single"]), 1) + self.assertEqual(row["Module"], "Core") From b1f6152f3886c3e53fcad435113282be3e78987e Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:19:14 +0100 Subject: [PATCH 47/72] fix: pop from form_params --- frappe/desk/reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 8428f13af5..5450d77377 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -345,7 +345,7 @@ def export_query(): form_params["as_list"] = True doctype = form_params.pop("doctype") file_format_type = form_params.pop("file_format_type") - title = frappe.form_dict.pop("title", doctype) + title = form_params.pop("title", doctype) csv_params = pop_csv_params(form_params) add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None From abb9055be4d715381a113483b2affb0bcc4ccada Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:53:34 +0530 Subject: [PATCH 48/72] fix(UX): Make fetch_from read_only if fetch_is_empty is not set (#19025) * fix(UX): Make fetch_from read_only if fetch_is_empty is not set * Update frappe/public/js/frappe/form/controls/base_input.js Co-authored-by: Ankush Menat
---
.../public/js/frappe/form/controls/base_input.js | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/frappe/public/js/frappe/form/controls/base_input.js b/frappe/public/js/frappe/form/controls/base_input.js
index 62f1c51f67..43fb4f54dc 100644
--- a/frappe/public/js/frappe/form/controls/base_input.js
+++ b/frappe/public/js/frappe/form/controls/base_input.js
@@ -83,7 +83,9 @@ frappe.ui.form.ControlInput = class ControlInput extends frappe.ui.form.Control
me.value = me.doc[me.df.fieldname] || "";
}
- if (me.can_write()) {
+ let is_fetch_from_read_only = me.df.fetch_from && !me.df.fetch_if_empty;
+
+ if (me.can_write() && !is_fetch_from_read_only) {
me.disp_area && $(me.disp_area).toggle(false);
$(me.input_area).toggle(true);
me.$input && me.$input.prop("disabled", false);
@@ -101,6 +103,16 @@ frappe.ui.form.ControlInput = class ControlInput extends frappe.ui.form.Control
}
}
me.$input && me.$input.prop("disabled", true);
+
+ if (is_fetch_from_read_only) {
+ $(me.disp_area).attr(
+ "title",
+ __(
+ "This value is fetched from {0}'s {1} field",
+ me.df.fetch_from.split(".")
+ )
+ );
+ }
}
me.set_description();
From 6dda853278866723a1d45c243b3d3018d7c56927 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Mon, 28 Nov 2022 20:46:51 +0530
Subject: [PATCH 49/72] fix: broken link for email tracking pixels (#19030)
---
frappe/email/doctype/email_queue/email_queue.py | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py
index 3c020eea39..56f7f6f5ea 100644
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@@ -26,6 +26,7 @@ from frappe.utils import (
cstr,
get_hook_method,
get_string_between,
+ get_url,
nowdate,
sbool,
split_emails,
@@ -293,15 +294,11 @@ class SendMailContext:
message = self.include_attachments(message)
return message
- def get_tracker_str(self):
- tracker_url_html = '
'
-
- message = ""
+ def get_tracker_str(self) -> str:
if frappe.conf.use_ssl and self.email_account_doc.track_email_status:
- message = quopri.encodestring(
- tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
- ).decode()
- return message
+ tracker_url_html = f'}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={self.queue_doc.communication})
'
+ return quopri.encodestring(tracker_url_html.encode()).decode()
+ return ""
def get_unsubscribe_str(self, recipient_email: str) -> str:
unsubscribe_url = ""
From c89cdb4f0c4a783c912da1f51e938ade21eb74c2 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Tue, 29 Nov 2022 11:30:12 +0530
Subject: [PATCH 50/72] test: handle multi-queue consumers
---
frappe/core/doctype/rq_worker/test_rq_worker.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/frappe/core/doctype/rq_worker/test_rq_worker.py b/frappe/core/doctype/rq_worker/test_rq_worker.py
index 5a43270681..f07338d630 100644
--- a/frappe/core/doctype/rq_worker/test_rq_worker.py
+++ b/frappe/core/doctype/rq_worker/test_rq_worker.py
@@ -10,7 +10,7 @@ class TestRQWorker(FrappeTestCase):
def test_get_worker_list(self):
workers = RQWorker.get_list({})
self.assertGreaterEqual(len(workers), 1)
- self.assertTrue(any(w.queue_type == "short" for w in workers))
+ self.assertTrue(any("short" in w.queue_type for w in workers))
def test_worker_serialization(self):
workers = RQWorker.get_list({})
From f25358ad6302473f80f27977e63091c5ff8dba33 Mon Sep 17 00:00:00 2001
From: HarryPaulo
Date: Tue, 29 Nov 2022 03:13:23 -0300
Subject: [PATCH 51/72] fix: "_dict" is an invalid attribute name because it
starts with "_" (#19010)
* fix: "_dict" is an invalid attribute name because it starts with "_"
* chore: move to frappe namespace
---
frappe/utils/safe_exec.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py
index 78c205d947..890f16a1c3 100644
--- a/frappe/utils/safe_exec.py
+++ b/frappe/utils/safe_exec.py
@@ -106,7 +106,6 @@ def get_safe_globals():
as_json=frappe.as_json,
dict=dict,
log=frappe.log,
- _dict=frappe._dict,
args=form_dict,
frappe=NamespaceDict(
call=call_whitelisted_function,
@@ -117,6 +116,7 @@ def get_safe_globals():
time_format=time_format,
format_date=frappe.utils.data.global_date_format,
form_dict=form_dict,
+ as_dict=frappe._dict,
bold=frappe.bold,
copy_doc=frappe.copy_doc,
errprint=frappe.errprint,
From 465b3f696822b3854cea9abfdcd30ce44b0a6b29 Mon Sep 17 00:00:00 2001
From: Ankush Menat
Date: Tue, 29 Nov 2022 11:53:52 +0530
Subject: [PATCH 52/72] test: use _dict in tests
form_dict is usually _dict, otherwise some unrelated tests will fail.
---
frappe/tests/test_query_report.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/frappe/tests/test_query_report.py b/frappe/tests/test_query_report.py
index 434eefb5d5..f93ab81c8b 100644
--- a/frappe/tests/test_query_report.py
+++ b/frappe/tests/test_query_report.py
@@ -89,14 +89,16 @@ class TestQueryReport(FrappeTestCase):
for delimiter in (",", ";", "\t", "|"):
for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC):
- frappe.local.form_dict = {
- "report_name": REPORT_NAME,
- "file_format_type": "CSV",
- "csv_quoting": quoting,
- "csv_delimiter": delimiter,
- "include_indentation": 0,
- "visible_idx": [0, 1, 2],
- }
+ frappe.local.form_dict = frappe._dict(
+ {
+ "report_name": REPORT_NAME,
+ "file_format_type": "CSV",
+ "csv_quoting": quoting,
+ "csv_delimiter": delimiter,
+ "include_indentation": 0,
+ "visible_idx": [0, 1, 2],
+ }
+ )
export_query()
self.assertTrue(frappe.response["filename"].endswith(".csv"))
From 9bfc43c1480dcef3b6d169970ae26787b73b1da0 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Fri, 18 Nov 2022 19:51:35 +0530
Subject: [PATCH 53/72] fix: use after_insert hook for enqueuing report
generation
this removes the need of additionally calling enqueue_report method
* enqueue report in long queue
* reduce the timeout from 100 mins to 30 mins
---
frappe/core/doctype/prepared_report/prepared_report.py | 6 ++++--
frappe/desk/query_report.py | 2 --
frappe/public/js/frappe/views/reports/query_report.js | 1 -
3 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 0b8e25229d..c6ed6452cf 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -18,8 +18,10 @@ class PreparedReport(Document):
self.status = "Queued"
self.report_start_time = frappe.utils.now()
- def enqueue_report(self):
- enqueue(run_background, prepared_report=self.name, timeout=6000)
+ def after_insert(self):
+ enqueue(
+ run_background, queue="long", prepared_report=self.name, timeout=1800, enqueue_after_commit=True
+ )
def run_background(prepared_report):
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 8b08352a04..0f5ba3beab 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -165,8 +165,6 @@ def background_enqueue_run(report_name, filters=None, user=None):
}
)
track_instance.insert(ignore_permissions=True)
- frappe.db.commit()
- track_instance.enqueue_report()
return {
"name": track_instance.name,
diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 7f20a69efc..ba7859b9a5 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -867,7 +867,6 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
return new Promise((resolve) =>
frappe.call({
method: "frappe.desk.query_report.background_enqueue_run",
- type: "GET",
args: {
report_name: this.report_name,
filters: filters,
From 4fe8eadf5e503529ed6540ee54b6c9275e83a378 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Fri, 18 Nov 2022 20:29:32 +0530
Subject: [PATCH 54/72] refactor: remove unnecessary fields from prepared
report
* removed ref_report_doctype
* changed title_field from ref_report_doctype to report_name (both are same)
* changed report_name field to link type
* changed autonaming to hash based
* added status in list view via document states
---
.../prepared_report/prepared_report.json | 48 +++++++++----------
.../prepared_report/prepared_report.py | 11 ++---
.../prepared_report/prepared_report_list.js | 12 -----
.../prepared_report/test_prepared_report.py | 1 -
frappe/core/doctype/report/report.py | 2 +-
frappe/desk/query_report.py | 18 ++-----
6 files changed, 33 insertions(+), 59 deletions(-)
delete mode 100644 frappe/core/doctype/prepared_report/prepared_report_list.js
diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json
index cafe323519..de4e95c717 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.json
+++ b/frappe/core/doctype/prepared_report/prepared_report.json
@@ -1,36 +1,28 @@
{
"actions": [],
- "autoname": "REP.#####",
+ "autoname": "hash",
"creation": "2018-06-25 18:39:11.152960",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
- "report_name",
- "ref_report_doctype",
"status",
+ "report_name",
"column_break_4",
"report_start_time",
+ "column_break_5",
"report_end_time",
"section_break_7",
"error_message",
"filters_sb",
"filters",
- "filter_values",
- "columns"
+ "filter_values"
],
"fields": [
{
"fieldname": "report_name",
- "fieldtype": "Data",
- "label": "Report Name",
- "read_only": 1
- },
- {
- "fieldname": "ref_report_doctype",
"fieldtype": "Link",
- "in_standard_filter": 1,
- "label": "Report Type",
+ "label": "Report Name",
"options": "Report",
"read_only": 1
},
@@ -92,22 +84,17 @@
"label": "Filter Values"
},
{
- "fieldname": "columns",
- "fieldtype": "Code",
- "hidden": 1,
- "label": "Columns",
- "no_copy": 1,
- "print_hide": 1,
- "read_only": 1
+ "fieldname": "column_break_5",
+ "fieldtype": "Column Break"
}
],
"in_create": 1,
"links": [],
- "modified": "2022-06-13 06:20:34.496412",
+ "modified": "2022-11-18 20:22:27.539535",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
- "naming_rule": "Expression (old style)",
+ "naming_rule": "Random",
"owner": "Administrator",
"permissions": [
{
@@ -134,7 +121,20 @@
],
"sort_field": "modified",
"sort_order": "DESC",
- "states": [],
- "title_field": "ref_report_doctype",
+ "states": [
+ {
+ "color": "Blue",
+ "title": "Queued"
+ },
+ {
+ "color": "Red",
+ "title": "Error"
+ },
+ {
+ "color": "Green",
+ "title": "Completed"
+ }
+ ],
+ "title_field": "report_name",
"track_changes": 1
}
\ No newline at end of file
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index c6ed6452cf..3e6368703d 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -26,9 +26,9 @@ class PreparedReport(Document):
def run_background(prepared_report):
instance = frappe.get_doc("Prepared Report", prepared_report)
- report = frappe.get_doc("Report", instance.ref_report_doctype)
+ report = frappe.get_doc("Report", instance.report_name)
- add_data_to_monitor(report=instance.ref_report_doctype)
+ add_data_to_monitor(report=instance.report_name)
try:
report.custom_columns = []
@@ -46,7 +46,6 @@ def run_background(prepared_report):
create_json_gz_file(result["result"], "Prepared Report", instance.name)
instance.status = "Completed"
- instance.columns = json.dumps(result["columns"])
instance.report_end_time = frappe.utils.now()
instance.save(ignore_permissions=True)
@@ -151,9 +150,7 @@ def get_permission_query_condition(user):
reports = [frappe.db.escape(report) for report in user.get_all_reports().keys()]
- return """`tabPrepared Report`.ref_report_doctype in ({reports})""".format(
- reports=",".join(reports)
- )
+ return """`tabPrepared Report`.report_name in ({reports})""".format(reports=",".join(reports))
def has_permission(doc, user):
@@ -169,4 +166,4 @@ def has_permission(doc, user):
if "System Manager" in user.roles:
return True
- return doc.ref_report_doctype in user.get_all_reports().keys()
+ return doc.report_name in user.get_all_reports().keys()
diff --git a/frappe/core/doctype/prepared_report/prepared_report_list.js b/frappe/core/doctype/prepared_report/prepared_report_list.js
deleted file mode 100644
index 1414dd7580..0000000000
--- a/frappe/core/doctype/prepared_report/prepared_report_list.js
+++ /dev/null
@@ -1,12 +0,0 @@
-frappe.listview_settings["Prepared Report"] = {
- add_fields: ["status"],
- get_indicator: function (doc) {
- if (doc.status === "Completed") {
- return [__("Completed"), "green", "status,=,Completed"];
- } else if (doc.status === "Error") {
- return [__("Error"), "red", "status,=,Error"];
- } else if (doc.status === "Queued") {
- return [__("Queued"), "orange", "status,=,Queued"];
- }
- },
-};
diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py
index cd96e8a18f..9426cebff4 100644
--- a/frappe/core/doctype/prepared_report/test_prepared_report.py
+++ b/frappe/core/doctype/prepared_report/test_prepared_report.py
@@ -15,7 +15,6 @@ class TestPreparedReport(FrappeTestCase):
"doctype": "Prepared Report",
"report_name": self.report.name,
"filters": json.dumps(self.filters),
- "ref_report_doctype": self.report.name,
}
).insert()
diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 7fe3cadf9c..a732a0c512 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -61,7 +61,7 @@ class Report(Document):
def delete_prepared_reports(self):
prepared_reports = frappe.get_all(
- "Prepared Report", filters={"ref_report_doctype": self.name}, pluck="name"
+ "Prepared Report", filters={"report_name": self.name}, pluck="name"
)
for report in prepared_reports:
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 0f5ba3beab..32714c2ab5 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -146,30 +146,20 @@ def normalize_result(result, columns):
@frappe.whitelist()
-def background_enqueue_run(report_name, filters=None, user=None):
+def background_enqueue_run(report_name, filters=None):
"""run reports in background"""
- if not user:
- user = frappe.session.user
- report = get_report_doc(report_name)
- track_instance = frappe.get_doc(
+ prepared_report = frappe.get_doc(
{
"doctype": "Prepared Report",
"report_name": report_name,
# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
# We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
"filters": json.dumps(json.loads(filters)),
- "ref_report_doctype": report_name,
- "report_type": report.report_type,
- "query": report.query,
- "module": report.module,
}
)
- track_instance.insert(ignore_permissions=True)
+ prepared_report.insert(ignore_permissions=True)
- return {
- "name": track_instance.name,
- "redirect_url": get_url_to_form("Prepared Report", track_instance.name),
- }
+ return {"name": prepared_report.name}
@frappe.whitelist()
From ecbafb5a23b7ac0141979169691f9e2b46407c2a Mon Sep 17 00:00:00 2001
From: phot0n
Date: Fri, 18 Nov 2022 21:17:33 +0530
Subject: [PATCH 55/72] feat(minor): add job_id column to prepared report
* chore: make report_name mandatory
---
.../prepared_report/prepared_report.json | 13 ++++++++-----
.../doctype/prepared_report/prepared_report.py | 17 +++++++++++++----
2 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json
index de4e95c717..4d7fab4e1f 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.json
+++ b/frappe/core/doctype/prepared_report/prepared_report.json
@@ -8,9 +8,9 @@
"field_order": [
"status",
"report_name",
+ "job_id",
"column_break_4",
"report_start_time",
- "column_break_5",
"report_end_time",
"section_break_7",
"error_message",
@@ -24,7 +24,8 @@
"fieldtype": "Link",
"label": "Report Name",
"options": "Report",
- "read_only": 1
+ "read_only": 1,
+ "reqd": 1
},
{
"default": "Queued",
@@ -84,13 +85,15 @@
"label": "Filter Values"
},
{
- "fieldname": "column_break_5",
- "fieldtype": "Column Break"
+ "fieldname": "job_id",
+ "fieldtype": "Data",
+ "label": "Job ID",
+ "read_only": 1
}
],
"in_create": 1,
"links": [],
- "modified": "2022-11-18 20:22:27.539535",
+ "modified": "2022-11-18 21:05:24.197945",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 3e6368703d..a6a8d89845 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -4,6 +4,8 @@
import json
+from rq import get_current_job
+
import frappe
from frappe.desk.form.load import get_attachments
from frappe.desk.query_report import generate_report_result
@@ -20,13 +22,22 @@ class PreparedReport(Document):
def after_insert(self):
enqueue(
- run_background, queue="long", prepared_report=self.name, timeout=1800, enqueue_after_commit=True
+ generate_report,
+ queue="long",
+ prepared_report=self.name,
+ timeout=1800,
+ enqueue_after_commit=True,
)
+ def update_job_id(self, job_id):
+ frappe.db.set_value(self.doctype, self.name, "job_id", job_id, update_modified=False)
+ frappe.db.commit()
-def run_background(prepared_report):
+
+def generate_report(prepared_report):
instance = frappe.get_doc("Prepared Report", prepared_report)
report = frappe.get_doc("Report", instance.report_name)
+ instance.update_job_id(get_current_job().id)
add_data_to_monitor(report=instance.report_name)
@@ -50,8 +61,6 @@ def run_background(prepared_report):
instance.save(ignore_permissions=True)
except Exception:
- report.log_error("Prepared report failed")
- instance = frappe.get_doc("Prepared Report", prepared_report)
instance.status = "Error"
instance.error_message = frappe.get_traceback()
instance.save(ignore_permissions=True)
From cd468c3229fbe395d7b4ec8cb668023e70a1d670 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Fri, 18 Nov 2022 21:57:21 +0530
Subject: [PATCH 56/72] fix(ux): don't show report_name field on prepared
report doc
---
.../core/doctype/prepared_report/prepared_report.js | 5 +++++
.../doctype/prepared_report/prepared_report.json | 12 +++++++++---
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.js b/frappe/core/doctype/prepared_report/prepared_report.js
index 58f4c1957f..c7ee7bdf63 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.js
+++ b/frappe/core/doctype/prepared_report/prepared_report.js
@@ -30,6 +30,11 @@ frappe.ui.form.on("Prepared Report", {
refresh: function (frm) {
frm.disable_save();
+
+ // always keep report_name hidden - we do this as we can't set mandatory and hidden
+ // property on a docfield at the same time
+ frm.toggle_display(["report_name"], 0);
+
if (frm.doc.status == "Completed") {
frm.page.set_primary_action(__("Show Report"), () => {
frappe.set_route(
diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json
index 4d7fab4e1f..f99a7d41d0 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.json
+++ b/frappe/core/doctype/prepared_report/prepared_report.json
@@ -8,10 +8,11 @@
"field_order": [
"status",
"report_name",
- "job_id",
- "column_break_4",
"report_start_time",
+ "column_break_4",
"report_end_time",
+ "column_break_6",
+ "job_id",
"section_break_7",
"error_message",
"filters_sb",
@@ -88,12 +89,17 @@
"fieldname": "job_id",
"fieldtype": "Data",
"label": "Job ID",
+ "no_copy": 1,
"read_only": 1
+ },
+ {
+ "fieldname": "column_break_6",
+ "fieldtype": "Column Break"
}
],
"in_create": 1,
"links": [],
- "modified": "2022-11-18 21:05:24.197945",
+ "modified": "2022-11-18 21:52:45.444209",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
From 6218a993016c1180565f67507a0a0e5fdc70aee6 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Sat, 19 Nov 2022 00:35:13 +0530
Subject: [PATCH 57/72] refactor(minor): dry-er code in generate_report
---
frappe/core/doctype/prepared_report/prepared_report.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index a6a8d89845..2f31cd5139 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -36,8 +36,8 @@ class PreparedReport(Document):
def generate_report(prepared_report):
instance = frappe.get_doc("Prepared Report", prepared_report)
- report = frappe.get_doc("Report", instance.report_name)
instance.update_job_id(get_current_job().id)
+ report = frappe.get_doc("Report", instance.report_name)
add_data_to_monitor(report=instance.report_name)
@@ -57,13 +57,12 @@ def generate_report(prepared_report):
create_json_gz_file(result["result"], "Prepared Report", instance.name)
instance.status = "Completed"
- instance.report_end_time = frappe.utils.now()
- instance.save(ignore_permissions=True)
-
except Exception:
instance.status = "Error"
instance.error_message = frappe.get_traceback()
- instance.save(ignore_permissions=True)
+
+ instance.report_end_time = frappe.utils.now()
+ instance.save(ignore_permissions=True)
frappe.publish_realtime(
"report_generated",
From 18d48ddeb8b916f22c663811ffc6ef40ecccb04b Mon Sep 17 00:00:00 2001
From: phot0n
Date: Sun, 20 Nov 2022 18:09:41 +0530
Subject: [PATCH 58/72] refactor: cleanup peprared result render and old logs
cleanup
* directly fetch columns from result file rather than storing it in db
* remove prepared report settings from system settings
* remove disable_prepared_report from report doctype
---
.../core/doctype/log_settings/log_settings.py | 2 +-
.../prepared_report/prepared_report.json | 5 +-
.../prepared_report/prepared_report.py | 62 ++++++++----
frappe/core/doctype/report/report.json | 13 +--
frappe/core/doctype/report/report.py | 16 +---
.../role_permission_for_page_and_report.py | 9 +-
.../system_settings/system_settings.json | 33 ++-----
frappe/desk/query_report.py | 96 ++++++-------------
frappe/hooks.py | 1 -
.../js/frappe/views/reports/query_report.js | 2 +-
10 files changed, 92 insertions(+), 147 deletions(-)
diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py
index f0f2cdaae8..e1e1d5d176 100644
--- a/frappe/core/doctype/log_settings/log_settings.py
+++ b/frappe/core/doctype/log_settings/log_settings.py
@@ -18,6 +18,7 @@ DEFAULT_LOGTYPES_RETENTION = {
"Scheduled Job Log": 90,
"Route History": 90,
"Submission Queue": 30,
+ "Prepared Report": 30,
}
@@ -155,7 +156,6 @@ LOG_DOCTYPES = [
"Email Queue Recipient",
"Error Snapshot",
"Error Log",
- "Submission Queue",
]
diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json
index f99a7d41d0..c3b21e42a9 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.json
+++ b/frappe/core/doctype/prepared_report/prepared_report.json
@@ -22,9 +22,8 @@
"fields": [
{
"fieldname": "report_name",
- "fieldtype": "Link",
+ "fieldtype": "Data",
"label": "Report Name",
- "options": "Report",
"read_only": 1,
"reqd": 1
},
@@ -99,7 +98,7 @@
],
"in_create": 1,
"links": [],
- "modified": "2022-11-18 21:52:45.444209",
+ "modified": "2022-11-20 15:15:00.907626",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 2f31cd5139..cff9296558 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -16,6 +16,16 @@ from frappe.utils.background_jobs import enqueue
class PreparedReport(Document):
+ @staticmethod
+ def clear_old_logs(days=30):
+ prepared_reports_to_delete = frappe.get_all(
+ "Prepared Report",
+ filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]},
+ )
+
+ for batch in frappe.utils.create_batch(prepared_reports_to_delete, 100):
+ enqueue(method=delete_prepared_reports, reports=batch)
+
def before_insert(self):
self.status = "Queued"
self.report_start_time = frappe.utils.now()
@@ -33,6 +43,11 @@ class PreparedReport(Document):
frappe.db.set_value(self.doctype, self.name, "job_id", job_id, update_modified=False)
frappe.db.commit()
+ def get_prepared_data(self, attachment_name=None):
+ if attached_file_name := attachment_name or get_attachments(self.doctype, self.name)[0]:
+ attached_file = frappe.get_doc("File", attached_file_name)
+ return gzip_decompress(attached_file.get_content())
+
def generate_report(prepared_report):
instance = frappe.get_doc("Prepared Report", prepared_report)
@@ -54,7 +69,7 @@ def generate_report(prepared_report):
report.custom_columns = data["columns"]
result = generate_report_result(report=report, filters=instance.filters, user=instance.owner)
- create_json_gz_file(result["result"], "Prepared Report", instance.name)
+ create_json_gz_file(result, instance.doctype, instance.name)
instance.status = "Completed"
except Exception:
@@ -71,6 +86,22 @@ def generate_report(prepared_report):
)
+@frappe.whitelist()
+def make_prepared_report(report_name, filters=None):
+ """run reports in background"""
+ prepared_report = frappe.get_doc(
+ {
+ "doctype": "Prepared Report",
+ "report_name": report_name,
+ # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
+ # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
+ "filters": json.dumps(json.loads(filters)),
+ }
+ ).insert(ignore_permissions=True)
+
+ return {"name": prepared_report.name}
+
+
@frappe.whitelist()
def get_reports_in_queued_state(report_name, filters):
reports = frappe.get_all(
@@ -79,27 +110,22 @@ def get_reports_in_queued_state(report_name, filters):
"report_name": report_name,
"filters": json.dumps(json.loads(filters)),
"status": "Queued",
+ "owner": frappe.session.user,
},
)
return reports
-def delete_expired_prepared_reports():
- system_settings = frappe.get_single("System Settings")
- enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion
- if enable_auto_deletion:
- expiry_period = system_settings.prepared_report_expiry_period
- prepared_reports_to_delete = frappe.get_all(
- "Prepared Report",
- filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -expiry_period)]},
- )
-
- batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
- for batch in batches:
- args = {
- "reports": batch,
- }
- enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
+def get_completed_prepared_report(filters, user, report_name):
+ return frappe.db.get_value(
+ "Prepared Report",
+ filters={
+ "status": "Completed",
+ "filters": json.dumps(filters),
+ "owner": user,
+ "report_name": report_name,
+ },
+ )
@frappe.whitelist()
@@ -137,7 +163,7 @@ def create_json_gz_file(data, dt, dn):
@frappe.whitelist()
def download_attachment(dn):
attachment = get_attachments("Prepared Report", dn)[0]
- frappe.local.response.filename = attachment.file_name[:-2]
+ frappe.local.response.filename = attachment.file_name[:-3]
attached_file = frappe.get_doc("File", attachment.name)
frappe.local.response.filecontent = gzip_decompress(attached_file.get_content())
frappe.local.response.type = "binary"
diff --git a/frappe/core/doctype/report/report.json b/frappe/core/doctype/report/report.json
index 4b98f1a3eb..37dce73dda 100644
--- a/frappe/core/doctype/report/report.json
+++ b/frappe/core/doctype/report/report.json
@@ -16,7 +16,6 @@
"letter_head",
"add_total_row",
"disabled",
- "disable_prepared_report",
"prepared_report",
"filters_section",
"filters",
@@ -133,19 +132,11 @@
"label": "Roles",
"options": "Has Role"
},
- {
- "default": "0",
- "fieldname": "disable_prepared_report",
- "fieldtype": "Check",
- "label": "Disable Prepared Report"
- },
{
"default": "0",
"fieldname": "prepared_report",
"fieldtype": "Check",
- "hidden": 1,
- "label": "Prepared Report",
- "read_only": 1
+ "label": "Prepared Report"
},
{
"depends_on": "eval:(doc.report_type===\"Script Report\" \n|| doc.report_type==\"Query Report\") \n&& doc.is_standard===\"No\"",
@@ -191,7 +182,7 @@
"idx": 1,
"index_web_pages_for_search": 1,
"links": [],
- "modified": "2022-09-15 13:37:24.531848",
+ "modified": "2022-11-20 14:56:36.578412",
"modified_by": "Administrator",
"module": "Core",
"name": "Report",
diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index a732a0c512..ae862184b6 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -57,17 +57,6 @@ class Report(Document):
):
frappe.throw(_("You are not allowed to delete Standard Report"))
delete_custom_role("report", self.name)
- self.delete_prepared_reports()
-
- def delete_prepared_reports(self):
- prepared_reports = frappe.get_all(
- "Prepared Report", filters={"report_name": self.name}, pluck="name"
- )
-
- for report in prepared_reports:
- frappe.delete_doc(
- "Prepared Report", report, ignore_missing=True, force=True, delete_permanently=True
- )
def get_columns(self):
return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
@@ -344,9 +333,8 @@ class Report(Document):
self.db_set("disabled", cint(disable))
-@frappe.whitelist()
-def is_prepared_report_disabled(report):
- return frappe.db.get_value("Report", report, "disable_prepared_report") or 0
+def is_prepared_report_enabled(report):
+ return cint(frappe.db.get_value("Report", report, "prepared_report")) or 0
def get_report_module_dotted_path(module, report_name):
diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
index bd61995ba3..17ccf35171 100644
--- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
+++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
@@ -2,8 +2,9 @@
# License: MIT. See LICENSE
import frappe
-from frappe.core.doctype.report.report import is_prepared_report_disabled
+from frappe.core.doctype.report.report import is_prepared_report_enabled
from frappe.model.document import Document
+from frappe.utils import cint
class RolePermissionforPageandReport(Document):
@@ -27,7 +28,7 @@ class RolePermissionforPageandReport(Document):
def check_prepared_report_disabled(self):
if self.report:
- self.disable_prepared_report = is_prepared_report_disabled(self.report)
+ self.disable_prepared_report = not is_prepared_report_enabled(self.report)
def get_standard_roles(self):
doctype = self.set_role_for
@@ -67,9 +68,9 @@ class RolePermissionforPageandReport(Document):
if self.report:
# intentionally written update query in frappe.db.sql instead of frappe.db.set_value
frappe.db.sql(
- """ update `tabReport` set disable_prepared_report = %s
+ """update `tabReport` set prepared_report = %s
where name = %s""",
- (self.disable_prepared_report, self.report),
+ (1 - cint(self.disable_prepared_report), self.report),
)
def get_args(self, row=None):
diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json
index 5fd5ef8163..57e2b1a1be 100644
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@@ -70,9 +70,6 @@
"hide_footer_in_auto_email_reports",
"attach_view_link",
"prepared_report_section",
- "enable_prepared_report_auto_deletion",
- "prepared_report_expiry_period",
- "column_break_64",
"max_auto_email_report_per_user",
"system_updates_section",
"disable_system_update_notification",
@@ -427,25 +424,11 @@
"label": "Send document Web View link in email"
},
{
- "default": "30",
- "depends_on": "enable_prepared_report_auto_deletion",
- "description": "System will auto-delete Prepared Reports permanently after these many days since creation",
- "fieldname": "prepared_report_expiry_period",
- "fieldtype": "Int",
- "label": "Prepared Report Expiry Period (Days)"
- },
- {
- "default": "1",
- "fieldname": "enable_prepared_report_auto_deletion",
- "fieldtype": "Check",
- "label": "Enable Auto-deletion of Prepared Reports"
- },
- {
- "collapsible": 1,
- "fieldname": "prepared_report_section",
- "fieldtype": "Section Break",
- "label": "Reports"
- },
+ "collapsible": 1,
+ "fieldname": "prepared_report_section",
+ "fieldtype": "Section Break",
+ "label": "Reports"
+ },
{
"default": "Frappe",
"description": "The application name will be used in the Login page.",
@@ -498,10 +481,6 @@
"fieldtype": "Check",
"label": "Allow Older Web View Links (Insecure)"
},
- {
- "fieldname": "column_break_64",
- "fieldtype": "Column Break"
- },
{
"default": "20",
"fieldname": "max_auto_email_report_per_user",
@@ -538,7 +517,7 @@
"icon": "fa fa-cog",
"issingle": 1,
"links": [],
- "modified": "2022-10-30 12:02:46.639170",
+ "modified": "2022-11-20 17:57:05.099512",
"modified_by": "Administrator",
"module": "Core",
"name": "System Settings",
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 32714c2ab5..2a1ccaff98 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -145,23 +145,6 @@ def normalize_result(result, columns):
return data
-@frappe.whitelist()
-def background_enqueue_run(report_name, filters=None):
- """run reports in background"""
- prepared_report = frappe.get_doc(
- {
- "doctype": "Prepared Report",
- "report_name": report_name,
- # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
- # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
- "filters": json.dumps(json.loads(filters)),
- }
- )
- prepared_report.insert(ignore_permissions=True)
-
- return {"name": prepared_report.name}
-
-
@frappe.whitelist()
def get_script(report_name):
report = get_report_doc(report_name)
@@ -219,12 +202,7 @@ def run(
result = None
- if (
- report.prepared_report
- and not report.disable_prepared_report
- and not ignore_prepared_report
- and not custom_columns
- ):
+ if report.prepared_report and not ignore_prepared_report and not custom_columns:
if filters:
if isinstance(filters, str):
filters = json.loads(filters)
@@ -260,57 +238,41 @@ def add_custom_column_data(custom_columns, result):
def get_prepared_report_result(report, filters, dn="", user=None):
- latest_report_data = {}
- doc = None
- if dn:
- # Get specified dn
- doc = frappe.get_doc("Prepared Report", dn)
- else:
- # Only look for completed prepared reports with given filters.
- doc_list = frappe.get_all(
- "Prepared Report",
- filters={
- "status": "Completed",
- "filters": json.dumps(filters),
- "owner": user,
- "report_name": report.get("custom_report") or report.get("report_name"),
- },
- order_by="creation desc",
+ from frappe.core.doctype.prepared_report.prepared_report import get_completed_prepared_report
+
+ def get_report_data(doc, data):
+ # backwards compatibility - prepared report used to have a columns field,
+ # we now directly fetch it from the result file
+ if isinstance(data, list):
+ columns = (doc.get("columns") and json.loads(doc.columns)) or data[0]
+ result = data
+ else:
+ columns = data.get("columns")
+ result = data.get("result")
+
+ for column in columns:
+ if isinstance(column, dict) and column.get("label"):
+ column["label"] = _(column["label"])
+
+ return {"columns": columns, "result": result}
+
+ report_data = {}
+ if not dn:
+ dn = get_completed_prepared_report(
+ filters, user, report.get("custom_report") or report.get("report_name")
)
- if doc_list:
- # Get latest
- doc = frappe.get_doc("Prepared Report", doc_list[0])
-
+ doc = frappe.get_doc("Prepared Report", dn) if dn else None
if doc:
try:
- # Prepared Report data is stored in a GZip compressed JSON file
- attached_file_name = frappe.db.get_value(
- "File",
- {"attached_to_doctype": doc.doctype, "attached_to_name": doc.name},
- "name",
- )
- attached_file = frappe.get_doc("File", attached_file_name)
- compressed_content = attached_file.get_content()
- uncompressed_content = gzip_decompress(compressed_content)
- data = json.loads(uncompressed_content.decode("utf-8"))
- if data:
- columns = json.loads(doc.columns) if doc.columns else data[0]
-
- for column in columns:
- if isinstance(column, dict) and column.get("label"):
- column["label"] = _(column["label"])
-
- latest_report_data = {"columns": columns, "result": data}
+ if data := json.loads(doc.get_prepared_data().decode("utf-8")):
+ report_data = get_report_data(doc, data)
except Exception:
- doc.log_error("Prepared report failed")
- frappe.delete_doc("Prepared Report", doc.name)
- frappe.db.commit()
+ doc.log_error("Prepared report render failed")
+ frappe.msgprint(_("Prepared report render failed"))
doc = None
- latest_report_data.update({"prepared_report": True, "doc": doc})
-
- return latest_report_data
+ return report_data | {"prepared_report": True, "doc": doc}
@frappe.whitelist()
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 73327f0ab1..ce52048c04 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -220,7 +220,6 @@ scheduler_events = {
"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
- "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports",
"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",
"frappe.utils.subscription.enable_manage_subscription",
],
diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index ba7859b9a5..bcb35fad30 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -866,7 +866,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
let filters = this.get_filter_values(true);
return new Promise((resolve) =>
frappe.call({
- method: "frappe.desk.query_report.background_enqueue_run",
+ method: "frappe.core.doctype.prepared_report.prepared_report.make_prepared_report",
args: {
report_name: this.report_name,
filters: filters,
From 433615beed8c8b3628d635899c074079ef09911b Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 15:16:41 +0530
Subject: [PATCH 59/72] chore: add queued_by virual docfield in prepared report
---
.../prepared_report/prepared_report.json | 17 ++++++++++-------
.../doctype/prepared_report/prepared_report.py | 4 ++++
2 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json
index c3b21e42a9..3a89b7030b 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.json
+++ b/frappe/core/doctype/prepared_report/prepared_report.json
@@ -8,11 +8,11 @@
"field_order": [
"status",
"report_name",
- "report_start_time",
- "column_break_4",
- "report_end_time",
- "column_break_6",
+ "queued_by",
"job_id",
+ "column_break_4",
+ "report_start_time",
+ "report_end_time",
"section_break_7",
"error_message",
"filters_sb",
@@ -92,13 +92,16 @@
"read_only": 1
},
{
- "fieldname": "column_break_6",
- "fieldtype": "Column Break"
+ "fieldname": "queued_by",
+ "fieldtype": "Data",
+ "is_virtual": 1,
+ "label": "Queued By",
+ "read_only": 1
}
],
"in_create": 1,
"links": [],
- "modified": "2022-11-20 15:15:00.907626",
+ "modified": "2022-11-22 15:14:40.205678",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index cff9296558..49c53fb5ff 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -16,6 +16,10 @@ from frappe.utils.background_jobs import enqueue
class PreparedReport(Document):
+ @property
+ def queued_by(self):
+ return self.owner
+
@staticmethod
def clear_old_logs(days=30):
prepared_reports_to_delete = frappe.get_all(
From 08bae70908c11af73ad797f70d3e4c60bc0de248 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 15:19:19 +0530
Subject: [PATCH 60/72] perf: use modified for fetching old logs for clearing
---
frappe/core/doctype/prepared_report/prepared_report.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 49c53fb5ff..b129266dbd 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -24,7 +24,7 @@ class PreparedReport(Document):
def clear_old_logs(days=30):
prepared_reports_to_delete = frappe.get_all(
"Prepared Report",
- filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]},
+ filters={"modified": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]},
)
for batch in frappe.utils.create_batch(prepared_reports_to_delete, 100):
From 917dfdae78b799e9bbc59e4e83958c124ea09d0d Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 15:25:03 +0530
Subject: [PATCH 61/72] refactor(minor): use virtual docfield for
report_start_time - and renamed it to queued_at
* chore: rename Report End Time label to Finished At
---
.../prepared_report/prepared_report.json | 19 ++++++++++---------
.../prepared_report/prepared_report.py | 5 ++++-
.../prepared_report/test_prepared_report.py | 2 +-
3 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json
index 3a89b7030b..6379bb15db 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.json
+++ b/frappe/core/doctype/prepared_report/prepared_report.json
@@ -11,7 +11,7 @@
"queued_by",
"job_id",
"column_break_4",
- "report_start_time",
+ "queued_at",
"report_end_time",
"section_break_7",
"error_message",
@@ -42,16 +42,10 @@
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
- {
- "fieldname": "report_start_time",
- "fieldtype": "Datetime",
- "label": "Report Start Time",
- "read_only": 1
- },
{
"fieldname": "report_end_time",
"fieldtype": "Datetime",
- "label": "Report End Time",
+ "label": "Finished At",
"read_only": 1
},
{
@@ -97,11 +91,18 @@
"is_virtual": 1,
"label": "Queued By",
"read_only": 1
+ },
+ {
+ "fieldname": "queued_at",
+ "fieldtype": "Datetime",
+ "is_virtual": 1,
+ "label": "Queued At",
+ "read_only": 1
}
],
"in_create": 1,
"links": [],
- "modified": "2022-11-22 15:14:40.205678",
+ "modified": "2022-11-22 15:22:45.523427",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index b129266dbd..64a68fdc3e 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -20,6 +20,10 @@ class PreparedReport(Document):
def queued_by(self):
return self.owner
+ @property
+ def queued_at(self):
+ return self.creation
+
@staticmethod
def clear_old_logs(days=30):
prepared_reports_to_delete = frappe.get_all(
@@ -32,7 +36,6 @@ class PreparedReport(Document):
def before_insert(self):
self.status = "Queued"
- self.report_start_time = frappe.utils.now()
def after_insert(self):
enqueue(
diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py
index 9426cebff4..6fd189b960 100644
--- a/frappe/core/doctype/prepared_report/test_prepared_report.py
+++ b/frappe/core/doctype/prepared_report/test_prepared_report.py
@@ -24,4 +24,4 @@ class TestPreparedReport(FrappeTestCase):
def test_for_creation(self):
self.assertTrue("QUEUED" == self.prepared_report_doc.status.upper())
- self.assertTrue(self.prepared_report_doc.report_start_time)
+ self.assertTrue(self.prepared_report_doc.queued_at)
From eea2fd97829f7288e31d55ab638d009b8f973a7a Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 15:40:01 +0530
Subject: [PATCH 62/72] fix: dont show filters when not present in prepared
report
---
frappe/core/doctype/prepared_report/prepared_report.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.js b/frappe/core/doctype/prepared_report/prepared_report.js
index c7ee7bdf63..7a0053b319 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.js
+++ b/frappe/core/doctype/prepared_report/prepared_report.js
@@ -16,6 +16,7 @@ frappe.ui.form.on("Prepared Report", {
`);
const filters = JSON.parse(frm.doc.filters);
+ frm.toggle_display(["filter_values"], !$.isEmptyObject(filters));
Object.keys(filters).forEach((key) => {
const filter_row = $(`
From 0d275b68f2283683f120e4f226b1ad8491897403 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 16:35:38 +0530
Subject: [PATCH 63/72] fix: get_prepared_data - add with_file_name param for
returning filename of the attachment
---
.../doctype/prepared_report/prepared_report.py | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 64a68fdc3e..12c5e0d512 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -50,9 +50,13 @@ class PreparedReport(Document):
frappe.db.set_value(self.doctype, self.name, "job_id", job_id, update_modified=False)
frappe.db.commit()
- def get_prepared_data(self, attachment_name=None):
- if attached_file_name := attachment_name or get_attachments(self.doctype, self.name)[0]:
- attached_file = frappe.get_doc("File", attached_file_name)
+ def get_prepared_data(self, with_file_name=False):
+ if attachments := get_attachments(self.doctype, self.name):
+ attachment = attachments[0]
+ attached_file = frappe.get_doc("File", attachment.name)
+
+ if with_file_name:
+ return (gzip_decompress(attached_file.get_content()), attachment.file_name)
return gzip_decompress(attached_file.get_content())
@@ -169,10 +173,9 @@ def create_json_gz_file(data, dt, dn):
@frappe.whitelist()
def download_attachment(dn):
- attachment = get_attachments("Prepared Report", dn)[0]
- frappe.local.response.filename = attachment.file_name[:-3]
- attached_file = frappe.get_doc("File", attachment.name)
- frappe.local.response.filecontent = gzip_decompress(attached_file.get_content())
+ data, file_name = frappe.get_doc("Prepared Report", dn).get_prepared_data(with_file_name=True)
+ frappe.local.response.filename = file_name[:-3]
+ frappe.local.response.filecontent = data
frappe.local.response.type = "binary"
From bf132ab2a877c8e404c096640c24b2835982d17a Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 17:15:33 +0530
Subject: [PATCH 64/72] fix: use doc.get to check if columns field exists or
not
---
frappe/desk/query_report.py | 14 +++-----------
1 file changed, 3 insertions(+), 11 deletions(-)
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 2a1ccaff98..7d72763a0f 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -15,15 +15,7 @@ from frappe.model.utils import render_include
from frappe.modules import get_module_path, scrub
from frappe.monitor import add_data_to_monitor
from frappe.permissions import get_role_permissions
-from frappe.utils import (
- cint,
- cstr,
- flt,
- format_duration,
- get_html_format,
- get_url_to_form,
- gzip_decompress,
-)
+from frappe.utils import cint, cstr, flt, format_duration, get_html_format
def get_report_doc(report_name):
@@ -243,8 +235,8 @@ def get_prepared_report_result(report, filters, dn="", user=None):
def get_report_data(doc, data):
# backwards compatibility - prepared report used to have a columns field,
# we now directly fetch it from the result file
- if isinstance(data, list):
- columns = (doc.get("columns") and json.loads(doc.columns)) or data[0]
+ if doc.get("columns"):
+ columns = json.loads(doc.columns) or data[0]
result = data
else:
columns = data.get("columns")
From a69c82e06c8737a64024634522290cae1edc61b9 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 18:30:34 +0530
Subject: [PATCH 65/72] feat(minor): option to show charts, total_row with
prepared reports
---
frappe/desk/query_report.py | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 7d72763a0f..a1e8a9368f 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -199,8 +199,7 @@ def run(
if isinstance(filters, str):
filters = json.loads(filters)
- dn = filters.get("prepared_report_name")
- filters.pop("prepared_report_name", None)
+ dn = filters.pop("prepared_report_name", None)
else:
dn = ""
result = get_prepared_report_result(report, filters, dn, user)
@@ -235,18 +234,17 @@ def get_prepared_report_result(report, filters, dn="", user=None):
def get_report_data(doc, data):
# backwards compatibility - prepared report used to have a columns field,
# we now directly fetch it from the result file
- if doc.get("columns"):
- columns = json.loads(doc.columns) or data[0]
- result = data
+ if doc.get("columns") or isinstance(data, list):
+ columns = (doc.get("columns") and json.loads(doc.columns)) or data[0]
+ data = {"result": data}
else:
columns = data.get("columns")
- result = data.get("result")
for column in columns:
if isinstance(column, dict) and column.get("label"):
column["label"] = _(column["label"])
- return {"columns": columns, "result": result}
+ return data | {"columns": columns}
report_data = {}
if not dn:
From 0370c80e41e4a1e69f42b29f8978047dee6dfa03 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 19:38:48 +0530
Subject: [PATCH 66/72] chore: patches for copying disable_prepared_report and
removing prepared report settings from system settings
---
frappe/patches.txt | 2 ++
..._disable_prepared_report_to_prepared_report.py | 6 ++++++
...epared_report_settings_from_system_settings.py | 15 +++++++++++++++
3 files changed, 23 insertions(+)
create mode 100644 frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py
create mode 100644 frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py
diff --git a/frappe/patches.txt b/frappe/patches.txt
index e8289a2790..9c298557c3 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -195,6 +195,8 @@ frappe.patches.v14_0.setup_likes_from_feedback
frappe.patches.v14_0.update_webforms
frappe.patches.v14_0.delete_payment_gateways
frappe.patches.v15_0.remove_event_streaming
+frappe.patches.v15_0.remove_prepared_report_settings_from_system_settings
+frappe.patches.v15_0.copy_disable_prepared_report_to_prepared_report
[post_model_sync]
frappe.patches.v14_0.drop_data_import_legacy
diff --git a/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py b/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py
new file mode 100644
index 0000000000..f1e4afb9a4
--- /dev/null
+++ b/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py
@@ -0,0 +1,6 @@
+import frappe
+
+
+def execute():
+ table = frappe.qb.DocType("Report")
+ frappe.qb.update(table).set(table.prepared_report, 0).where(table.disable_prepared_report == 1)
diff --git a/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py b/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py
new file mode 100644
index 0000000000..8c0ec4ca70
--- /dev/null
+++ b/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py
@@ -0,0 +1,15 @@
+import frappe
+from frappe.utils import cint
+
+
+def execute():
+ expiry_period = (
+ cint(frappe.db.get_single_value("System Settings", "prepared_report_expiry_period")) or 30
+ )
+ frappe.get_single("Log Settings").register_doctype("Prepared Report", expiry_period)
+
+ singles = frappe.qb.DocType("Singles")
+ frappe.qb.from_(singles).delete().where(
+ (singles.doctype == "System Settings")
+ & (singles.field.isin(["enable_prepared_report_auto_deletion", "prepared_report_expiry_period"]))
+ ).run()
From 246caea70e36decaac2c38ad3f44f46523a19924 Mon Sep 17 00:00:00 2001
From: phot0n
Date: Tue, 22 Nov 2022 23:41:42 +0530
Subject: [PATCH 67/72] test: prepared report tests
* fix: update job_id for prepared report
---
.../prepared_report/prepared_report.py | 12 +++--
.../prepared_report/test_prepared_report.py | 50 ++++++++++++++-----
2 files changed, 45 insertions(+), 17 deletions(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 12c5e0d512..5bbc23fb1e 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -46,10 +46,6 @@ class PreparedReport(Document):
enqueue_after_commit=True,
)
- def update_job_id(self, job_id):
- frappe.db.set_value(self.doctype, self.name, "job_id", job_id, update_modified=False)
- frappe.db.commit()
-
def get_prepared_data(self, with_file_name=False):
if attachments := get_attachments(self.doctype, self.name):
attachment = attachments[0]
@@ -61,8 +57,9 @@ class PreparedReport(Document):
def generate_report(prepared_report):
+ update_job_id(prepared_report, get_current_job().id)
+
instance = frappe.get_doc("Prepared Report", prepared_report)
- instance.update_job_id(get_current_job().id)
report = frappe.get_doc("Report", instance.report_name)
add_data_to_monitor(report=instance.report_name)
@@ -97,6 +94,11 @@ def generate_report(prepared_report):
)
+def update_job_id(prepared_report, job_id):
+ frappe.db.set_value("Prepared Report", prepared_report, "job_id", job_id, update_modified=False)
+ frappe.db.commit()
+
+
@frappe.whitelist()
def make_prepared_report(report_name, filters=None):
"""run reports in background"""
diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py
index 6fd189b960..a864ea73f8 100644
--- a/frappe/core/doctype/prepared_report/test_prepared_report.py
+++ b/frappe/core/doctype/prepared_report/test_prepared_report.py
@@ -1,27 +1,53 @@
# Copyright (c) 2018, Frappe Technologies and Contributors
# License: MIT. See LICENSE
import json
+import time
import frappe
+from frappe.desk.query_report import generate_report_result, get_report_doc
from frappe.tests.utils import FrappeTestCase
class TestPreparedReport(FrappeTestCase):
- def setUp(self):
- self.report = frappe.get_doc({"doctype": "Report", "name": "Permitted Documents For User"})
- self.filters = {"user": "Administrator", "doctype": "Role"}
- self.prepared_report_doc = frappe.get_doc(
+ @classmethod
+ def tearDownClass(cls):
+ for r in frappe.get_all("Prepared Report", pluck="name"):
+ frappe.delete_doc("Prepared Report", r, force=True, delete_permanently=True)
+
+ frappe.db.commit()
+
+ def create_prepared_report(self, commit=False):
+ doc = frappe.get_doc(
{
"doctype": "Prepared Report",
- "report_name": self.report.name,
- "filters": json.dumps(self.filters),
+ "report_name": "Database Storage Usage By Tables",
}
).insert()
- def tearDown(self):
- frappe.set_user("Administrator")
- self.prepared_report_doc.delete()
+ if commit:
+ frappe.db.commit()
- def test_for_creation(self):
- self.assertTrue("QUEUED" == self.prepared_report_doc.status.upper())
- self.assertTrue(self.prepared_report_doc.queued_at)
+ return doc
+
+ def test_queueing(self):
+ doc_ = self.create_prepared_report()
+ self.assertEqual("Queued", doc_.status)
+ self.assertTrue(doc_.queued_at)
+
+ frappe.db.commit()
+ time.sleep(5)
+
+ doc_ = frappe.get_last_doc("Prepared Report")
+ self.assertEqual("Completed", doc_.status)
+ self.assertTrue(doc_.job_id)
+ self.assertTrue(doc_.report_end_time)
+
+ def test_prepared_data(self):
+ doc_ = self.create_prepared_report(commit=True)
+ time.sleep(5)
+
+ prepared_data = json.loads(doc_.get_prepared_data().decode("utf-8"))
+ generated_data = generate_report_result(get_report_doc("Database Storage Usage By Tables"))
+ self.assertEqual(len(prepared_data["columns"]), len(generated_data["columns"]))
+ self.assertEqual(len(prepared_data["result"]), len(generated_data["result"]))
+ self.assertEqual(len(prepared_data), len(generated_data))
From c8ea0a72d08a591888ee8bc2a4f2a68023a017be Mon Sep 17 00:00:00 2001
From: phot0n
Date: Wed, 23 Nov 2022 00:18:54 +0530
Subject: [PATCH 68/72] fix: render filters on refresh
onload hook gave inconsistent behaviour
---
frappe/core/doctype/prepared_report/prepared_report.js | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/frappe/core/doctype/prepared_report/prepared_report.js b/frappe/core/doctype/prepared_report/prepared_report.js
index 7a0053b319..11f9caf8fd 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.js
+++ b/frappe/core/doctype/prepared_report/prepared_report.js
@@ -2,7 +2,7 @@
// For license information, please see license.txt
frappe.ui.form.on("Prepared Report", {
- onload: function (frm) {
+ render_filter_values: function (frm) {
var wrapper = $(frm.fields_dict["filter_values"].wrapper).empty();
let filter_table = $(`
@@ -31,6 +31,7 @@ frappe.ui.form.on("Prepared Report", {
refresh: function (frm) {
frm.disable_save();
+ frm.events.render_filter_values(frm);
// always keep report_name hidden - we do this as we can't set mandatory and hidden
// property on a docfield at the same time
From 1a5019432da293a5bc92e9b0251e5a5cbff66212 Mon Sep 17 00:00:00 2001
From: phot0n
-
-
-
-
+
@@ -1150,7 +1149,7 @@ export default class Grid {
const $wrapper = position === "top" ? this.grid_custom_buttons : this.grid_buttons;
let $btn = this.custom_buttons[label];
if (!$btn) {
- $btn = $(``)
+ $btn = $(``)
.prependTo($wrapper)
.on("click", click);
this.custom_buttons[label] = $btn;
diff --git a/frappe/public/scss/common/grid.scss b/frappe/public/scss/common/grid.scss
index ac15894670..43c45ff3b5 100644
--- a/frappe/public/scss/common/grid.scss
+++ b/frappe/public/scss/common/grid.scss
@@ -398,16 +398,21 @@
}
}
-.grid-buttons {
+.grid-buttons, .grid-bulk-actions {
display: inline-flex;
}
-.grid-footer {
+.grid-footer, .grid-custom-buttons {
padding: var(--padding-sm) 0px;
background-color: var(--fg-color);
.btn {
box-shadow: none;
margin-top: -3px;
+ margin-bottom: -3px;
+ }
+
+ .btn:not(:last-child) {
+ margin-right: 4px;
}
}
From a14367f1d98c05aef062ebe0f88fba1a11ccc1b4 Mon Sep 17 00:00:00 2001
From: Sagar Vora " + msg) doc.notify_update() + + +def bulk_insert( + doctype: str, + documents: Iterable["Document"], + ignore_duplicates: bool = False, + chunk_size=10_000, +): + """Insert simple Documents objects to database in bulk. + + Warning/Info: + - All documents are inserted without triggering ANY hooks. + - This function assumes you've done the due dilligence and inserts in similar fashion as db_insert + - Documents can be any iterable / generator containing Document objects + """ + + columns = frappe.get_meta(doctype).get_valid_columns() + values = _document_values_generator(documents, columns) + + frappe.db.bulk_insert( + doctype, columns, values, ignore_duplicates=ignore_duplicates, chunk_size=chunk_size + ) + + +def _document_values_generator( + documents: Iterable["Document"], + columns: list[str], +) -> Generator[tuple[Any], None, None]: + for doc in documents: + doc.creation = doc.modified = now() + doc.created_by = doc.modified_by = frappe.session.user + doc_values = doc.get_valid_dict( + convert_dates_to_str=True, + ignore_nulls=True, + ignore_virtual=True, + ) + yield tuple(doc_values.get(col) for col in columns) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 5a9c2d906d..ce30011d02 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -230,7 +230,7 @@ class Meta(Document): return fields - def get_valid_columns(self): + def get_valid_columns(self) -> list[str]: if not hasattr(self, "_valid_columns"): table_exists = frappe.db.table_exists(self.name) if self.name in self.special_doctypes and table_exists: diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 3833e911a7..4ad2d85a19 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -469,3 +469,21 @@ class TestDocumentWebView(FrappeTestCase): # Logged-in user can access the page without key self.assertEqual(self.get(url_without_key, "Administrator").status, "200 OK") + + def test_bulk_inserts(self): + from frappe.model.document import bulk_insert + + doctype = "ToDo" + sent_todo = set() + + def doc_generator(): + for i in range(690): + doc = frappe.new_doc(doctype) + doc.name = doc.description = frappe.generate_hash() + sent_todo.add(doc.name) + yield doc + + bulk_insert(doctype, doc_generator(), chunk_size=100) + + all_todos = set(frappe.get_all("ToDo", pluck="name")) + self.assertEqual(sent_todo - all_todos, set(), "All docs should be inserted") From 5008bc0b093b162e43f51cf1bdb130dc3704d7cf Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:15:30 +0100 Subject: [PATCH 44/72] fix: fieldname referenced before assignment --- frappe/desk/reportview.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 9adbeee290..3e5ee0bc92 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -425,16 +425,12 @@ def get_labels(fields, doctype): """get column labels based on column names""" labels = [] for key in fields: - key = key.split(" as ")[0] - - if key.startswith(("count(", "sum(", "avg(")): + try: + parenttype, fieldname = parse_field(key) + except ValueError: continue - if "." in key: - parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") - else: - parenttype = doctype - fieldname = fieldname.strip("`") + parenttype = parenttype or doctype if parenttype == doctype and fieldname == "name": label = _("ID", context="Label of name column in report") @@ -453,17 +449,12 @@ def get_labels(fields, doctype): def handle_duration_fieldtype_values(doctype, data, fields): for field in fields: - key = field.split(" as ")[0] - - if key.startswith(("count(", "sum(", "avg(")): + try: + parenttype, fieldname = parse_field(field) + except ValueError: continue - if "." in key: - parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") - else: - parenttype = doctype - fieldname = field.strip("`") - + parenttype = parenttype or doctype df = frappe.get_meta(parenttype).get_field(fieldname) if df and df.fieldtype == "Duration": @@ -476,6 +467,19 @@ def handle_duration_fieldtype_values(doctype, data, fields): return data +def parse_field(field: str) -> tuple[str | None, str]: + """Parse a field into parenttype and fieldname.""" + key = field.split(" as ")[0] + + if key.startswith(("count(", "sum(", "avg(")): + raise ValueError + + if "." in key: + return key.split(".")[0][4:-1], key.split(".")[1].strip("`") + + return None, key.strip("`") + + @frappe.whitelist() def delete_items(): """delete selected items""" From ee18984e1f389aecd44db5c21f512787b2141cc1 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:15:57 +0100 Subject: [PATCH 45/72] refactor: export_query --- frappe/desk/reportview.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 3e5ee0bc92..8428f13af5 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -340,31 +340,19 @@ def export_query(): """export from report builder""" from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file - title = frappe.form_dict.title - frappe.form_dict.pop("title", None) - form_params = get_form_params() form_params["limit_page_length"] = None form_params["as_list"] = True - doctype = form_params.doctype - add_totals_row = None - file_format_type = form_params["file_format_type"] - title = title or doctype + doctype = form_params.pop("doctype") + file_format_type = form_params.pop("file_format_type") + title = frappe.form_dict.pop("title", doctype) csv_params = pop_csv_params(form_params) - - del form_params["doctype"] - del form_params["file_format_type"] - - if "add_totals_row" in form_params and form_params["add_totals_row"] == "1": - add_totals_row = 1 - del form_params["add_totals_row"] + add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None frappe.permissions.can_export(doctype, raise_exception=True) - if "selected_items" in form_params: - si = json.loads(frappe.form_dict.get("selected_items")) - form_params["filters"] = {"name": ("in", si)} - del form_params["selected_items"] + if selection := form_params.pop("selected_items", None): + form_params["filters"] = {"name": ("in", json.loads(selection))} make_access_log( doctype=doctype, @@ -379,9 +367,8 @@ def export_query(): if add_totals_row: ret = append_totals_row(ret) - data = [[_("Sr")] + get_labels(db_query.fields, doctype)].extend( - [i + 1] + list(row) for i, row in enumerate(ret) - ) + data = [[_("Sr")] + get_labels(db_query.fields, doctype)] + data.extend([i + 1] + list(row) for i, row in enumerate(ret)) data = handle_duration_fieldtype_values(doctype, data, db_query.fields) if file_format_type == "CSV": From 2c0827dda95f14c620b80dd20030bba557afe990 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:16:18 +0100 Subject: [PATCH 46/72] fix: test reportview --- frappe/tests/test_reportview.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_reportview.py b/frappe/tests/test_reportview.py index ba3657aacd..4a24514acf 100644 --- a/frappe/tests/test_reportview.py +++ b/frappe/tests/test_reportview.py @@ -13,22 +13,22 @@ class TestReportview(FrappeTestCase): frappe.local.form_dict = frappe._dict( doctype="DocType", - title="Test Report", file_format_type="CSV", + fields=("name", "module", "issingle"), + filters={"issingle": 1, "module": "Core"}, ) for delimiter in (",", ";", "\t", "|"): + frappe.local.form_dict.csv_delimiter = delimiter for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC): - frappe.local.form_dict.update( - { - "csv_quoting": quoting, - "csv_delimiter": delimiter, - } - ) + frappe.local.form_dict.csv_quoting = quoting + export_query() self.assertTrue(frappe.response["filename"].endswith(".csv")) self.assertEqual(frappe.response["type"], "binary") with StringIO(frappe.response["filecontent"].decode("utf-8")) as result: reader = DictReader(result, delimiter=delimiter, quoting=quoting) - reader.__next__() + for row in reader: + self.assertEqual(int(row["Is Single"]), 1) + self.assertEqual(row["Module"], "Core") From b1f6152f3886c3e53fcad435113282be3e78987e Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:19:14 +0100 Subject: [PATCH 47/72] fix: pop from form_params --- frappe/desk/reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 8428f13af5..5450d77377 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -345,7 +345,7 @@ def export_query(): form_params["as_list"] = True doctype = form_params.pop("doctype") file_format_type = form_params.pop("file_format_type") - title = frappe.form_dict.pop("title", doctype) + title = form_params.pop("title", doctype) csv_params = pop_csv_params(form_params) add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None From abb9055be4d715381a113483b2affb0bcc4ccada Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:53:34 +0530 Subject: [PATCH 48/72] fix(UX): Make fetch_from read_only if fetch_is_empty is not set (#19025) * fix(UX): Make fetch_from read_only if fetch_is_empty is not set * Update frappe/public/js/frappe/form/controls/base_input.js Co-authored-by: Ankush Menat