vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'develop' into patch-1

Browse source
This commit is contained in:
Shariq Ansari 2022-11-29 15:12:19 +05:30 • committed by GitHub
commit a6d715b943
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
56 changed files with 1019 additions and 565 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frappe/__init__.py
View file

@ -1987,6 +1987,7 @@ def get_print(
no_letterhead=0,
password=None,
pdf_options=None,
letterhead=None,
):
"""Get Print Format for given document.
@ -2005,6 +2006,7 @@ def get_print(
local.form_dict.style = style
local.form_dict.doc = doc
local.form_dict.no_letterhead = no_letterhead
local.form_dict.letterhead = letterhead
pdf_options = pdf_options or {}
if password:

16
frappe/app.py
View file

@ -77,7 +77,7 @@ def application(request: Request):
rollback = after_request(rollback)
finally:
if request.method in ("POST", "PUT") and frappe.db and rollback:
if request.method in UNSAFE_HTTP_METHODS and frappe.db and rollback:
frappe.db.rollback()
frappe.rate_limiter.update()
@ -320,12 +320,16 @@ def handle_exception(e):
def after_request(rollback):
# if HTTP method would change server state, commit if necessary
if frappe.db and (
frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS
if (
frappe.db
and (frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS)
and frappe.db.transaction_writes
):
if frappe.db.transaction_writes:
frappe.db.commit()
rollback = False
frappe.db.commit()
rollback = False
elif frappe.db:
frappe.db.rollback()
rollback = False
# update session
if getattr(frappe.local, "session_obj", None):

27
frappe/commands/scheduler.py
View file

@ -178,15 +178,34 @@ def start_scheduler():
@click.command("worker")
@click.option("--queue", type=str)
@click.option(
"--queue",
type=str,
help="Queue to consume from. Multiple queues can be specified using comma-separated string. If not specified all queues are consumed.",
)
@click.option("--quiet", is_flag=True, default=False, help="Hide Log Outputs")
@click.option("-u", "--rq-username", default=None, help="Redis ACL user")
@click.option("-p", "--rq-password", default=None, help="Redis ACL user password")
def start_worker(queue, quiet=False, rq_username=None, rq_password=None):
"""Site is used to find redis credentals."""
@click.option("--burst", is_flag=True, default=False, help="Run Worker in Burst mode.")
@click.option(
"--strategy",
required=False,
type=click.Choice(["round_robin", "random"]),
help="Dequeuing strategy to use",
)
def start_worker(
queue, quiet=False, rq_username=None, rq_password=None, burst=False, strategy=None
):
from frappe.utils.background_jobs import start_worker
start_worker(queue, quiet=quiet, rq_username=rq_username, rq_password=rq_password)
start_worker(
queue,
quiet=quiet,
rq_username=rq_username,
rq_password=rq_password,
burst=burst,
strategy=strategy,
)
@click.command("ready-for-migration")

2
frappe/commands/site.py
View file

@ -1119,7 +1119,7 @@ def build_search_index(context):
@click.command("clear-log-table")
@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType")
@click.option("--doctype", required=True, type=click.Choice(LOG_DOCTYPES), help="Log DocType")
@click.option("--days", type=int, help="Keep records for days")
@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table")
@pass_context

11
frappe/core/doctype/data_export/exporter.py
View file

@ -432,23 +432,20 @@ class DataExporter:
row[_column_start_end.start + i + 1] = value
def build_response_as_excel(self):
from frappe.desk.utils import provide_binary_file
from frappe.utils.xlsxutils import make_xlsx
filename = frappe.generate_hash(length=10)
with open(filename, "wb") as f:
f.write(cstr(self.writer.getvalue()).encode("utf-8"))
f = open(filename)
reader = csv.reader(f)
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else "Data Export")
f.close()
os.remove(filename)
# write out response as a xlsx type
frappe.response["filename"] = self.doctype + ".xlsx"
frappe.response["filecontent"] = xlsx_file.getvalue()
frappe.response["type"] = "binary"
provide_binary_file(self.doctype, "xlsx", xlsx_file.getvalue())
def _append_name_column(self, dt=None):
self.append_field_column(

12
frappe/core/doctype/data_import/exporter.py
View file

@ -1,8 +1,6 @@
# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import typing
import frappe
from frappe import _
from frappe.model import display_fieldtypes, no_value_fields
@ -241,15 +239,9 @@ class Exporter:
def build_response(self):
if self.file_type == "CSV":
self.build_csv_response()
build_csv_response(self.get_csv_array_for_export(), _(self.doctype))
elif self.file_type == "Excel":
self.build_xlsx_response()
def build_csv_response(self):
build_csv_response(self.get_csv_array_for_export(), _(self.doctype))
def build_xlsx_response(self):
build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype))
build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype))
def group_children_data_by_parent(self, children_data: dict[str, list]):
return groupby_metric(children_data, key="parent")

2
frappe/core/doctype/doctype/doctype.js
View file

@ -124,6 +124,7 @@ frappe.ui.form.on("DocField", {
let doctypes = frm.doc.fields
.filter((df) => df.fieldtype == "Link")
.filter((df) => df.options && df.fieldname != row.fieldname)
.sort((a, b) => a.options.localeCompare(b.options))
.map((df) => ({
label: `${df.options} (${df.fieldname})`,
value: df.fieldname,
@ -151,6 +152,7 @@ frappe.ui.form.on("DocField", {
.get_docfields(link_doctype, null, {
fieldtype: ["not in", frappe.model.no_value_type],
})
.sort((a, b) => a.label.localeCompare(b.label))
.map((df) => ({
label: `${df.label} (${df.fieldtype})`,
value: df.fieldname,

2
frappe/core/doctype/log_settings/log_settings.py
View file

@ -18,6 +18,7 @@ DEFAULT_LOGTYPES_RETENTION = {
"Scheduled Job Log": 90,
"Route History": 90,
"Submission Queue": 30,
"Prepared Report": 30,
}
@ -155,7 +156,6 @@ LOG_DOCTYPES = [
"Email Queue Recipient",
"Error Snapshot",
"Error Log",
"Submission Queue",
]

9
frappe/core/doctype/prepared_report/prepared_report.js
View file

@ -2,7 +2,7 @@
// For license information, please see license.txt
frappe.ui.form.on("Prepared Report", {
onload: function (frm) {
render_filter_values: function (frm) {
var wrapper = $(frm.fields_dict["filter_values"].wrapper).empty();
let filter_table = $(`<table class="table table-bordered">
@ -16,6 +16,7 @@ frappe.ui.form.on("Prepared Report", {
</table>`);
const filters = JSON.parse(frm.doc.filters);
frm.toggle_display(["filter_values"], !$.isEmptyObject(filters));
Object.keys(filters).forEach((key) => {
const filter_row = $(`<tr>
@ -30,6 +31,12 @@ frappe.ui.form.on("Prepared Report", {
refresh: function (frm) {
frm.disable_save();
frm.events.render_filter_values(frm);
// always keep report_name hidden - we do this as we can't set mandatory and hidden
// property on a docfield at the same time
frm.toggle_display(["report_name"], 0);
if (frm.doc.status == "Completed") {
frm.page.set_primary_action(__("Show Report"), () => {
frappe.set_route(

75
frappe/core/doctype/prepared_report/prepared_report.json
View file

@ -1,38 +1,31 @@
{
"actions": [],
"autoname": "REP.#####",
"autoname": "hash",
"creation": "2018-06-25 18:39:11.152960",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"report_name",
"ref_report_doctype",
"status",
"report_name",
"queued_by",
"job_id",
"column_break_4",
"report_start_time",
"queued_at",
"report_end_time",
"section_break_7",
"error_message",
"filters_sb",
"filters",
"filter_values",
"columns"
"filter_values"
],
"fields": [
{
"fieldname": "report_name",
"fieldtype": "Data",
"label": "Report Name",
"read_only": 1
},
{
"fieldname": "ref_report_doctype",
"fieldtype": "Link",
"in_standard_filter": 1,
"label": "Report Type",
"options": "Report",
"read_only": 1
"read_only": 1,
"reqd": 1
},
{
"default": "Queued",
@ -49,16 +42,10 @@
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"fieldname": "report_start_time",
"fieldtype": "Datetime",
"label": "Report Start Time",
"read_only": 1
},
{
"fieldname": "report_end_time",
"fieldtype": "Datetime",
"label": "Report End Time",
"label": "Finished At",
"read_only": 1
},
{
@ -92,22 +79,35 @@
"label": "Filter Values"
},
{
"fieldname": "columns",
"fieldtype": "Code",
"hidden": 1,
"label": "Columns",
"fieldname": "job_id",
"fieldtype": "Link",
"label": "Job ID",
"no_copy": 1,
"print_hide": 1,
"options": "RQ Job",
"read_only": 1
},
{
"fieldname": "queued_by",
"fieldtype": "Data",
"is_virtual": 1,
"label": "Queued By",
"read_only": 1
},
{
"fieldname": "queued_at",
"fieldtype": "Datetime",
"is_virtual": 1,
"label": "Queued At",
"read_only": 1
}
],
"in_create": 1,
"links": [],
"modified": "2022-06-13 06:20:34.496412",
"modified": "2022-11-28 21:29:39.883803",
"modified_by": "Administrator",
"module": "Core",
"name": "Prepared Report",
"naming_rule": "Expression (old style)",
"naming_rule": "Random",
"owner": "Administrator",
"permissions": [
{
@ -134,7 +134,20 @@
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"title_field": "ref_report_doctype",
"states": [
{
"color": "Blue",
"title": "Queued"
},
{
"color": "Red",
"title": "Error"
},
{
"color": "Green",
"title": "Completed"
}
],
"title_field": "report_name",
"track_changes": 1
}

125
frappe/core/doctype/prepared_report/prepared_report.py
View file

@ -4,6 +4,8 @@
import json
from rq import get_current_job
import frappe
from frappe.desk.form.load import get_attachments
from frappe.desk.query_report import generate_report_result
@ -14,19 +16,53 @@ from frappe.utils.background_jobs import enqueue
class PreparedReport(Document):
@property
def queued_by(self):
return self.owner
@property
def queued_at(self):
return self.creation
@staticmethod
def clear_old_logs(days=30):
prepared_reports_to_delete = frappe.get_all(
"Prepared Report",
filters={"modified": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]},
)
for batch in frappe.utils.create_batch(prepared_reports_to_delete, 100):
enqueue(method=delete_prepared_reports, reports=batch)
def before_insert(self):
self.status = "Queued"
self.report_start_time = frappe.utils.now()
def enqueue_report(self):
enqueue(run_background, prepared_report=self.name, timeout=6000)
def after_insert(self):
enqueue(
generate_report,
queue="long",
prepared_report=self.name,
timeout=1500,
enqueue_after_commit=True,
)
def get_prepared_data(self, with_file_name=False):
if attachments := get_attachments(self.doctype, self.name):
attachment = attachments[0]
attached_file = frappe.get_doc("File", attachment.name)
if with_file_name:
return (gzip_decompress(attached_file.get_content()), attachment.file_name)
return gzip_decompress(attached_file.get_content())
def run_background(prepared_report):
def generate_report(prepared_report):
update_job_id(prepared_report, get_current_job().id)
instance = frappe.get_doc("Prepared Report", prepared_report)
report = frappe.get_doc("Report", instance.ref_report_doctype)
report = frappe.get_doc("Report", instance.report_name)
add_data_to_monitor(report=instance.ref_report_doctype)
add_data_to_monitor(report=instance.report_name)
try:
report.custom_columns = []
@ -41,19 +77,15 @@ def run_background(prepared_report):
report.custom_columns = data["columns"]
result = generate_report_result(report=report, filters=instance.filters, user=instance.owner)
create_json_gz_file(result["result"], "Prepared Report", instance.name)
create_json_gz_file(result, instance.doctype, instance.name)
instance.status = "Completed"
instance.columns = json.dumps(result["columns"])
instance.report_end_time = frappe.utils.now()
instance.save(ignore_permissions=True)
except Exception:
report.log_error("Prepared report failed")
instance = frappe.get_doc("Prepared Report", prepared_report)
instance.status = "Error"
instance.error_message = frappe.get_traceback()
instance.save(ignore_permissions=True)
instance.report_end_time = frappe.utils.now()
instance.save(ignore_permissions=True)
frappe.publish_realtime(
"report_generated",
@ -62,6 +94,27 @@ def run_background(prepared_report):
)
def update_job_id(prepared_report, job_id):
frappe.db.set_value("Prepared Report", prepared_report, "job_id", job_id, update_modified=False)
frappe.db.commit()
@frappe.whitelist()
def make_prepared_report(report_name, filters=None):
"""run reports in background"""
prepared_report = frappe.get_doc(
{
"doctype": "Prepared Report",
"report_name": report_name,
# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
# We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
"filters": json.dumps(json.loads(filters)),
}
).insert(ignore_permissions=True)
return {"name": prepared_report.name}
@frappe.whitelist()
def get_reports_in_queued_state(report_name, filters):
reports = frappe.get_all(
@ -70,27 +123,22 @@ def get_reports_in_queued_state(report_name, filters):
"report_name": report_name,
"filters": json.dumps(json.loads(filters)),
"status": "Queued",
"owner": frappe.session.user,
},
)
return reports
def delete_expired_prepared_reports():
system_settings = frappe.get_single("System Settings")
enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion
if enable_auto_deletion:
expiry_period = system_settings.prepared_report_expiry_period
prepared_reports_to_delete = frappe.get_all(
"Prepared Report",
filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -expiry_period)]},
)
batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
for batch in batches:
args = {
"reports": batch,
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
def get_completed_prepared_report(filters, user, report_name):
return frappe.db.get_value(
"Prepared Report",
filters={
"status": "Completed",
"filters": json.dumps(filters),
"owner": user,
"report_name": report_name,
},
)
@frappe.whitelist()
@ -127,10 +175,13 @@ def create_json_gz_file(data, dt, dn):
@frappe.whitelist()
def download_attachment(dn):
attachment = get_attachments("Prepared Report", dn)[0]
frappe.local.response.filename = attachment.file_name[:-2]
attached_file = frappe.get_doc("File", attachment.name)
frappe.local.response.filecontent = gzip_decompress(attached_file.get_content())
pr = frappe.get_doc("Prepared Report", dn)
if not pr.has_permission("read"):
frappe.throw(frappe._("Cannot Download Report due to insufficient permissions"))
data, file_name = pr.get_prepared_data(with_file_name=True)
frappe.local.response.filename = file_name[:-3]
frappe.local.response.filecontent = data
frappe.local.response.type = "binary"
@ -149,9 +200,7 @@ def get_permission_query_condition(user):
reports = [frappe.db.escape(report) for report in user.get_all_reports().keys()]
return """`tabPrepared Report`.ref_report_doctype in ({reports})""".format(
reports=",".join(reports)
)
return """`tabPrepared Report`.report_name in ({reports})""".format(reports=",".join(reports))
def has_permission(doc, user):
@ -167,4 +216,4 @@ def has_permission(doc, user):
if "System Manager" in user.roles:
return True
return doc.ref_report_doctype in user.get_all_reports().keys()
return doc.report_name in user.get_all_reports().keys()

12
frappe/core/doctype/prepared_report/prepared_report_list.js
View file

@ -1,12 +0,0 @@
frappe.listview_settings["Prepared Report"] = {
add_fields: ["status"],
get_indicator: function (doc) {
if (doc.status === "Completed") {
return [__("Completed"), "green", "status,=,Completed"];
} else if (doc.status === "Error") {
return [__("Error"), "red", "status,=,Error"];
} else if (doc.status === "Queued") {
return [__("Queued"), "orange", "status,=,Queued"];
}
},
};

51
frappe/core/doctype/prepared_report/test_prepared_report.py
View file

@ -1,28 +1,53 @@
# Copyright (c) 2018, Frappe Technologies and Contributors
# License: MIT. See LICENSE
import json
import time
import frappe
from frappe.desk.query_report import generate_report_result, get_report_doc
from frappe.tests.utils import FrappeTestCase
class TestPreparedReport(FrappeTestCase):
def setUp(self):
self.report = frappe.get_doc({"doctype": "Report", "name": "Permitted Documents For User"})
self.filters = {"user": "Administrator", "doctype": "Role"}
self.prepared_report_doc = frappe.get_doc(
@classmethod
def tearDownClass(cls):
for r in frappe.get_all("Prepared Report", pluck="name"):
frappe.delete_doc("Prepared Report", r, force=True, delete_permanently=True)
frappe.db.commit()
def create_prepared_report(self, commit=False):
doc = frappe.get_doc(
{
"doctype": "Prepared Report",
"report_name": self.report.name,
"filters": json.dumps(self.filters),
"ref_report_doctype": self.report.name,
"report_name": "Database Storage Usage By Tables",
}
).insert()
def tearDown(self):
frappe.set_user("Administrator")
self.prepared_report_doc.delete()
if commit:
frappe.db.commit()
def test_for_creation(self):
self.assertTrue("QUEUED" == self.prepared_report_doc.status.upper())
self.assertTrue(self.prepared_report_doc.report_start_time)
return doc
def test_queueing(self):
doc_ = self.create_prepared_report()
self.assertEqual("Queued", doc_.status)
self.assertTrue(doc_.queued_at)
frappe.db.commit()
time.sleep(5)
doc_ = frappe.get_last_doc("Prepared Report")
self.assertEqual("Completed", doc_.status)
self.assertTrue(doc_.job_id)
self.assertTrue(doc_.report_end_time)
def test_prepared_data(self):
doc_ = self.create_prepared_report(commit=True)
time.sleep(5)
prepared_data = json.loads(doc_.get_prepared_data().decode("utf-8"))
generated_data = generate_report_result(get_report_doc("Database Storage Usage By Tables"))
self.assertEqual(len(prepared_data["columns"]), len(generated_data["columns"]))
self.assertEqual(len(prepared_data["result"]), len(generated_data["result"]))
self.assertEqual(len(prepared_data), len(generated_data))

13
frappe/core/doctype/report/report.json
View file

@ -16,7 +16,6 @@
"letter_head",
"add_total_row",
"disabled",
"disable_prepared_report",
"prepared_report",
"filters_section",
"filters",
@ -133,19 +132,11 @@
"label": "Roles",
"options": "Has Role"
},
{
"default": "0",
"fieldname": "disable_prepared_report",
"fieldtype": "Check",
"label": "Disable Prepared Report"
},
{
"default": "0",
"fieldname": "prepared_report",
"fieldtype": "Check",
"hidden": 1,
"label": "Prepared Report",
"read_only": 1
"label": "Prepared Report"
},
{
"depends_on": "eval:(doc.report_type===\"Script Report\" \n|| doc.report_type==\"Query Report\") \n&& doc.is_standard===\"No\"",
@ -191,7 +182,7 @@
"idx": 1,
"index_web_pages_for_search": 1,
"links": [],
"modified": "2022-09-15 13:37:24.531848",
"modified": "2022-11-20 14:56:36.578412",
"modified_by": "Administrator",
"module": "Core",
"name": "Report",

16
frappe/core/doctype/report/report.py
View file

@ -57,17 +57,6 @@ class Report(Document):
):
frappe.throw(_("You are not allowed to delete Standard Report"))
delete_custom_role("report", self.name)
self.delete_prepared_reports()
def delete_prepared_reports(self):
prepared_reports = frappe.get_all(
"Prepared Report", filters={"ref_report_doctype": self.name}, pluck="name"
)
for report in prepared_reports:
frappe.delete_doc(
"Prepared Report", report, ignore_missing=True, force=True, delete_permanently=True
)
def get_columns(self):
return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
@ -344,9 +333,8 @@ class Report(Document):
self.db_set("disabled", cint(disable))
@frappe.whitelist()
def is_prepared_report_disabled(report):
return frappe.db.get_value("Report", report, "disable_prepared_report") or 0
def is_prepared_report_enabled(report):
return cint(frappe.db.get_value("Report", report, "prepared_report")) or 0
def get_report_module_dotted_path(module, report_name):

18
frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json
View file

@ -10,7 +10,7 @@
"page",
"report",
"column_break_4",
"disable_prepared_report",
"enable_prepared_report",
"roles_permission",
"roles_html",
"roles"
@ -42,13 +42,6 @@
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"default": "0",
"depends_on": "report",
"fieldname": "disable_prepared_report",
"fieldtype": "Check",
"label": "Disable Prepared Report"
},
{
"fieldname": "roles_permission",
"fieldtype": "Section Break",
@ -66,12 +59,19 @@
"label": "Roles",
"options": "Has Role",
"read_only": 1
},
{
"default": "0",
"depends_on": "report",
"fieldname": "enable_prepared_report",
"fieldtype": "Check",
"label": "Enable Prepared Report"
}
],
"hide_toolbar": 1,
"issingle": 1,
"links": [],
"modified": "2022-08-03 12:20:54.079809",
"modified": "2022-11-23 12:39:05.750386",
"modified_by": "Administrator",
"module": "Core",
"name": "Role Permission for Page and Report",

9
frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
View file

@ -2,8 +2,9 @@
# License: MIT. See LICENSE
import frappe
from frappe.core.doctype.report.report import is_prepared_report_disabled
from frappe.core.doctype.report.report import is_prepared_report_enabled
from frappe.model.document import Document
from frappe.utils import cint
class RolePermissionforPageandReport(Document):
@ -27,7 +28,7 @@ class RolePermissionforPageandReport(Document):
def check_prepared_report_disabled(self):
if self.report:
self.disable_prepared_report = is_prepared_report_disabled(self.report)
self.enable_prepared_report = is_prepared_report_enabled(self.report)
def get_standard_roles(self):
doctype = self.set_role_for
@ -67,9 +68,9 @@ class RolePermissionforPageandReport(Document):
if self.report:
# intentionally written update query in frappe.db.sql instead of frappe.db.set_value
frappe.db.sql(
""" update `tabReport` set disable_prepared_report = %s
"""update `tabReport` set prepared_report = %s
where name = %s""",
(self.disable_prepared_report, self.report),
(self.enable_prepared_report, self.report),
)
def get_args(self, row=None):

9
frappe/core/doctype/role_permission_for_page_and_report/test_role_permission_for_page_and_report.py Normal file
View file

@ -0,0 +1,9 @@
# Copyright (c) 2022, Frappe Technologies and Contributors
# See license.txt
# import frappe
from frappe.tests.utils import FrappeTestCase
class TestRolePermissionforPageandReport(FrappeTestCase):
pass

10
frappe/core/doctype/rq_job/test_rq_job.py
View file

@ -10,6 +10,7 @@ from rq.job import Job
import frappe
from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs, stop_job
from frappe.tests.utils import FrappeTestCase, timeout
from frappe.utils import cstr, execute_in_shell
from frappe.utils.background_jobs import is_job_queued
@ -92,6 +93,15 @@ class TestRQJob(FrappeTestCase):
self.check_status(actual_job, "finished")
self.assertFalse(is_job_queued(job_name))
@timeout(20)
def test_multi_queue_burst_consumption(self):
for _ in range(3):
for q in ["default", "short"]:
frappe.enqueue(self.BG_JOB, sleep=1, queue=q)
_, stderr = execute_in_shell("bench worker --queue short,default --burst", check_exit_code=True)
self.assertIn("quitting", cstr(stderr))
def test_func(fail=False, sleep=0):
if fail:

6
frappe/core/doctype/rq_worker/rq_worker.json
View file

@ -76,13 +76,13 @@
{
"fieldname": "queue",
"fieldtype": "Data",
"label": "Queue"
"label": "Queue(s)"
},
{
"fieldname": "queue_type",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Queue Type",
"label": "Queue Type(s)",
"options": "default\nlong\nshort"
},
{
@ -113,7 +113,7 @@
"in_create": 1,
"is_virtual": 1,
"links": [],
"modified": "2022-11-14 15:35:32.786012",
"modified": "2022-11-24 14:50:48.511706",
"modified_by": "Administrator",
"module": "Core",
"name": "RQ Worker",

13
frappe/core/doctype/rq_worker/rq_worker.py
View file

@ -16,8 +16,10 @@ class RQWorker(Document):
def load_from_db(self):
all_workers = get_workers()
worker = [w for w in all_workers if w.pid == cint(self.name)][0]
d = serialize_worker(worker)
workers = [w for w in all_workers if w.pid == cint(self.name)]
if not workers:
raise frappe.DoesNotExistError
d = serialize_worker(workers[0])
super(Document, self).__init__(d)
@ -51,12 +53,15 @@ class RQWorker(Document):
def serialize_worker(worker: Worker) -> frappe._dict:
queue = ", ".join(worker.queue_names())
queue_names = worker.queue_names()
queue = ", ".join(queue_names)
queue_types = ",".join(q.rsplit(":", 1)[1] for q in queue_names)
return frappe._dict(
name=worker.pid,
queue=queue,
queue_type=queue.rsplit(":", 1)[1],
queue_type=queue_types,
worker_name=worker.name,
status=worker.get_state(),
pid=worker.pid,

2
frappe/core/doctype/rq_worker/test_rq_worker.py
View file

@ -10,7 +10,7 @@ class TestRQWorker(FrappeTestCase):
def test_get_worker_list(self):
workers = RQWorker.get_list({})
self.assertGreaterEqual(len(workers), 1)
self.assertTrue(any(w.queue_type == "short" for w in workers))
self.assertTrue(any("short" in w.queue_type for w in workers))
def test_worker_serialization(self):
workers = RQWorker.get_list({})

33
frappe/core/doctype/system_settings/system_settings.json
View file

@ -70,9 +70,6 @@
"hide_footer_in_auto_email_reports",
"attach_view_link",
"prepared_report_section",
"enable_prepared_report_auto_deletion",
"prepared_report_expiry_period",
"column_break_64",
"max_auto_email_report_per_user",
"system_updates_section",
"disable_system_update_notification",
@ -427,25 +424,11 @@
"label": "Send document Web View link in email"
},
{
"default": "30",
"depends_on": "enable_prepared_report_auto_deletion",
"description": "System will auto-delete Prepared Reports permanently after these many days since creation",
"fieldname": "prepared_report_expiry_period",
"fieldtype": "Int",
"label": "Prepared Report Expiry Period (Days)"
},
{
"default": "1",
"fieldname": "enable_prepared_report_auto_deletion",
"fieldtype": "Check",
"label": "Enable Auto-deletion of Prepared Reports"
},
{
"collapsible": 1,
"fieldname": "prepared_report_section",
"fieldtype": "Section Break",
"label": "Reports"
},
"collapsible": 1,
"fieldname": "prepared_report_section",
"fieldtype": "Section Break",
"label": "Reports"
},
{
"default": "Frappe",
"description": "The application name will be used in the Login page.",
@ -498,10 +481,6 @@
"fieldtype": "Check",
"label": "Allow Older Web View Links (Insecure)"
},
{
"fieldname": "column_break_64",
"fieldtype": "Column Break"
},
{
"default": "20",
"fieldname": "max_auto_email_report_per_user",
@ -538,7 +517,7 @@
"icon": "fa fa-cog",
"issingle": 1,
"links": [],
"modified": "2022-10-30 12:02:46.639170",
"modified": "2022-11-20 17:57:05.099512",
"modified_by": "Administrator",
"module": "Core",
"name": "System Settings",

36
frappe/database/database.py
View file

@ -2,6 +2,7 @@
# License: MIT. See LICENSE
import datetime
import itertools
import json
import random
import re
@ -9,6 +10,7 @@ import string
import traceback
from contextlib import contextmanager, suppress
from time import time
from typing import Any, Iterable, Sequence
from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
from pypika.terms import Criterion, NullValue
@ -1204,28 +1206,36 @@ class Database:
frappe.flags.touched_tables = set()
frappe.flags.touched_tables.update(tables)
def bulk_insert(self, doctype, fields, values, ignore_duplicates=False, *, chunk_size=10_000):
def bulk_insert(
self,
doctype: str,
fields: list[str],
values: Iterable[Sequence[Any]],
ignore_duplicates=False,
*,
chunk_size=10_000,
):
"""
Insert multiple records at a time
:param doctype: Doctype name
:param fields: list of fields
:params values: list of list of values
:params values: iterable of values
"""
values = list(values)
table = frappe.qb.DocType(doctype)
for start_index in range(0, len(values), chunk_size):
query = frappe.qb.into(table)
if ignore_duplicates:
# Pypika does not have same api for ignoring duplicates
if self.db_type == "mariadb":
query = query.ignore()
elif self.db_type == "postgres":
query = query.on_conflict().do_nothing()
query = frappe.qb.into(table).columns(fields)
values_to_insert = values[start_index : start_index + chunk_size]
query.columns(fields).insert(*values_to_insert).run()
if ignore_duplicates:
# Pypika does not have same api for ignoring duplicates
if frappe.conf.db_type == "mariadb":
query = query.ignore()
elif frappe.conf.db_type == "postgres":
query = query.on_conflict().do_nothing()
value_iterator = iter(values)
while value_chunk := tuple(itertools.islice(value_iterator, chunk_size)):
query.insert(*value_chunk).run()
def create_sequence(self, *args, **kwargs):
from frappe.database.sequence import create_sequence

2
frappe/database/mariadb/database.py
View file

@ -119,7 +119,7 @@ class MariaDBConnectionUtil:
"use_unicode": True,
}
if self.user != "root":
if self.user not in (frappe.flags.root_login, "root"):
conn_settings["database"] = self.user
if self.port:

1
frappe/desk/form/linked_with.py
View file

@ -403,7 +403,6 @@ def get_exempted_doctypes():
return auto_cancel_exempt_doctypes
@frappe.whitelist()
def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> dict[str, list]:
if isinstance(linkinfo, str):
# additional fields are added in linkinfo

183
frappe/desk/query_report.py
View file

@ -10,19 +10,12 @@ import frappe
import frappe.desk.reportview
from frappe import _
from frappe.core.utils import ljust_list
from frappe.desk.reportview import clean_params, parse_json
from frappe.model.utils import render_include
from frappe.modules import get_module_path, scrub
from frappe.monitor import add_data_to_monitor
from frappe.permissions import get_role_permissions
from frappe.utils import (
cint,
cstr,
flt,
format_duration,
get_html_format,
get_url_to_form,
gzip_decompress,
)
from frappe.utils import cint, cstr, flt, format_duration, get_html_format
def get_report_doc(report_name):
@ -144,35 +137,6 @@ def normalize_result(result, columns):
return data
@frappe.whitelist()
def background_enqueue_run(report_name, filters=None, user=None):
"""run reports in background"""
if not user:
user = frappe.session.user
report = get_report_doc(report_name)
track_instance = frappe.get_doc(
{
"doctype": "Prepared Report",
"report_name": report_name,
# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
# We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
"filters": json.dumps(json.loads(filters)),
"ref_report_doctype": report_name,
"report_type": report.report_type,
"query": report.query,
"module": report.module,
}
)
track_instance.insert(ignore_permissions=True)
frappe.db.commit()
track_instance.enqueue_report()
return {
"name": track_instance.name,
"redirect_url": get_url_to_form("Prepared Report", track_instance.name),
}
@frappe.whitelist()
def get_script(report_name):
report = get_report_doc(report_name)
@ -230,18 +194,12 @@ def run(
result = None
if (
report.prepared_report
and not report.disable_prepared_report
and not ignore_prepared_report
and not custom_columns
):
if report.prepared_report and not ignore_prepared_report and not custom_columns:
if filters:
if isinstance(filters, str):
filters = json.loads(filters)
dn = filters.get("prepared_report_name")
filters.pop("prepared_report_name", None)
dn = filters.pop("prepared_report_name", None)
else:
dn = ""
result = get_prepared_report_result(report, filters, dn, user)
@ -271,103 +229,88 @@ def add_custom_column_data(custom_columns, result):
def get_prepared_report_result(report, filters, dn="", user=None):
latest_report_data = {}
doc = None
if dn:
# Get specified dn
doc = frappe.get_doc("Prepared Report", dn)
else:
# Only look for completed prepared reports with given filters.
doc_list = frappe.get_all(
"Prepared Report",
filters={
"status": "Completed",
"filters": json.dumps(filters),
"owner": user,
"report_name": report.get("custom_report") or report.get("report_name"),
},
order_by="creation desc",
from frappe.core.doctype.prepared_report.prepared_report import get_completed_prepared_report
def get_report_data(doc, data):
# backwards compatibility - prepared report used to have a columns field,
# we now directly fetch it from the result file
if doc.get("columns") or isinstance(data, list):
columns = (doc.get("columns") and json.loads(doc.columns)) or data[0]
data = {"result": data}
else:
columns = data.get("columns")
for column in columns:
if isinstance(column, dict) and column.get("label"):
column["label"] = _(column["label"])
return data | {"columns": columns}
report_data = {}
if not dn:
dn = get_completed_prepared_report(
filters, user, report.get("custom_report") or report.get("report_name")
)
if doc_list:
# Get latest
doc = frappe.get_doc("Prepared Report", doc_list[0])
doc = frappe.get_doc("Prepared Report", dn) if dn else None
if doc:
try:
# Prepared Report data is stored in a GZip compressed JSON file
attached_file_name = frappe.db.get_value(
"File",
{"attached_to_doctype": doc.doctype, "attached_to_name": doc.name},
"name",
)
attached_file = frappe.get_doc("File", attached_file_name)
compressed_content = attached_file.get_content()
uncompressed_content = gzip_decompress(compressed_content)
data = json.loads(uncompressed_content.decode("utf-8"))
if data:
columns = json.loads(doc.columns) if doc.columns else data[0]
for column in columns:
if isinstance(column, dict) and column.get("label"):
column["label"] = _(column["label"])
latest_report_data = {"columns": columns, "result": data}
if data := json.loads(doc.get_prepared_data().decode("utf-8")):
report_data = get_report_data(doc, data)
except Exception:
doc.log_error("Prepared report failed")
frappe.delete_doc("Prepared Report", doc.name)
frappe.db.commit()
doc.log_error("Prepared report render failed")
frappe.msgprint(_("Prepared report render failed"))
doc = None
latest_report_data.update({"prepared_report": True, "doc": doc})
return latest_report_data
return report_data | {"prepared_report": True, "doc": doc}
@frappe.whitelist()
def export_query():
"""export from query reports"""
data = frappe._dict(frappe.local.form_dict)
data.pop("cmd", None)
data.pop("csrf_token", None)
from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
if isinstance(data.get("filters"), str):
filters = json.loads(data["filters"])
form_params = frappe._dict(frappe.local.form_dict)
csv_params = pop_csv_params(form_params)
clean_params(form_params)
parse_json(form_params)
if data.get("report_name"):
report_name = data["report_name"]
frappe.permissions.can_export(
frappe.get_cached_value("Report", report_name, "ref_doctype"),
raise_exception=True,
)
report_name = form_params.report_name
frappe.permissions.can_export(
frappe.get_cached_value("Report", report_name, "ref_doctype"),
raise_exception=True,
)
file_format_type = data.get("file_format_type")
custom_columns = frappe.parse_json(data.get("custom_columns", "[]"))
include_indentation = data.get("include_indentation")
visible_idx = data.get("visible_idx")
file_format_type = form_params.file_format_type
custom_columns = frappe.parse_json(form_params.custom_columns or "[]")
include_indentation = form_params.include_indentation
visible_idx = form_params.visible_idx
if isinstance(visible_idx, str):
visible_idx = json.loads(visible_idx)
if file_format_type == "Excel":
data = run(report_name, filters, custom_columns=custom_columns)
data = frappe._dict(data)
if not data.columns:
frappe.respond_as_web_page(
_("No data to export"),
_("You can try changing the filters of your report."),
)
return
data = run(report_name, form_params.filters, custom_columns=custom_columns)
data = frappe._dict(data)
if not data.columns:
frappe.respond_as_web_page(
_("No data to export"),
_("You can try changing the filters of your report."),
)
return
format_duration_fields(data)
xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
if file_format_type == "CSV":
content = get_csv_bytes(xlsx_data, csv_params)
file_extension = "csv"
elif file_format_type == "Excel":
from frappe.utils.xlsxutils import make_xlsx
format_duration_fields(data)
xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
xlsx_file = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths)
file_extension = "xlsx"
content = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths).getvalue()
frappe.response["filename"] = report_name + ".xlsx"
frappe.response["filecontent"] = xlsx_file.getvalue()
frappe.response["type"] = "binary"
provide_binary_file(report_name, file_extension, content)
def format_duration_fields(data: frappe._dict) -> None:

98
frappe/desk/reportview.py
View file

@ -4,7 +4,6 @@
"""build query for doclistview and return results"""
import json
from io import StringIO
import frappe
import frappe.permissions
@ -14,7 +13,7 @@ from frappe.model import child_table_fields, default_fields, optional_fields
from frappe.model.base_document import get_controller
from frappe.model.db_query import DatabaseQuery
from frappe.model.utils import is_virtual_doctype
from frappe.utils import add_user_info, cstr, format_duration
from frappe.utils import add_user_info, format_duration
@frappe.whitelist()
@ -339,30 +338,21 @@ def delete_report(name):
@frappe.read_only()
def export_query():
"""export from report builder"""
title = frappe.form_dict.title
frappe.form_dict.pop("title", None)
from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
form_params = get_form_params()
form_params["limit_page_length"] = None
form_params["as_list"] = True
doctype = form_params.doctype
add_totals_row = None
file_format_type = form_params["file_format_type"]
title = title or doctype
del form_params["doctype"]
del form_params["file_format_type"]
if "add_totals_row" in form_params and form_params["add_totals_row"] == "1":
add_totals_row = 1
del form_params["add_totals_row"]
doctype = form_params.pop("doctype")
file_format_type = form_params.pop("file_format_type")
title = form_params.pop("title", doctype)
csv_params = pop_csv_params(form_params)
add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None
frappe.permissions.can_export(doctype, raise_exception=True)
if "selected_items" in form_params:
si = json.loads(frappe.form_dict.get("selected_items"))
form_params["filters"] = {"name": ("in", si)}
del form_params["selected_items"]
if selection := form_params.pop("selected_items", None):
form_params["filters"] = {"name": ("in", json.loads(selection))}
make_access_log(
doctype=doctype,
@ -378,38 +368,24 @@ def export_query():
ret = append_totals_row(ret)
data = [[_("Sr")] + get_labels(db_query.fields, doctype)]
for i, row in enumerate(ret):
data.append([i + 1] + list(row))
data.extend([i + 1] + list(row) for i, row in enumerate(ret))
data = handle_duration_fieldtype_values(doctype, data, db_query.fields)
if file_format_type == "CSV":
# convert to csv
import csv
from frappe.utils.xlsxutils import handle_html
f = StringIO()
writer = csv.writer(f)
for r in data:
# encode only unicode type strings and not int, floats etc.
writer.writerow([handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r])
f.seek(0)
frappe.response["result"] = cstr(f.read())
frappe.response["type"] = "csv"
frappe.response["doctype"] = title
file_extension = "csv"
content = get_csv_bytes(
[[handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r] for r in data],
csv_params,
)
elif file_format_type == "Excel":
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(data, doctype)
file_extension = "xlsx"
content = make_xlsx(data, doctype).getvalue()
frappe.response["filename"] = title + ".xlsx"
frappe.response["filecontent"] = xlsx_file.getvalue()
frappe.response["type"] = "binary"
provide_binary_file(title, file_extension, content)
def append_totals_row(data):
@ -436,16 +412,12 @@ def get_labels(fields, doctype):
"""get column labels based on column names"""
labels = []
for key in fields:
key = key.split(" as ")[0]
if key.startswith(("count(", "sum(", "avg(")):
try:
parenttype, fieldname = parse_field(key)
except ValueError:
continue
if "." in key:
parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
else:
parenttype = doctype
fieldname = fieldname.strip("`")
parenttype = parenttype or doctype
if parenttype == doctype and fieldname == "name":
label = _("ID", context="Label of name column in report")
@ -464,17 +436,12 @@ def get_labels(fields, doctype):
def handle_duration_fieldtype_values(doctype, data, fields):
for field in fields:
key = field.split(" as ")[0]
if key.startswith(("count(", "sum(", "avg(")):
try:
parenttype, fieldname = parse_field(field)
except ValueError:
continue
if "." in key:
parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
else:
parenttype = doctype
fieldname = field.strip("`")
parenttype = parenttype or doctype
df = frappe.get_meta(parenttype).get_field(fieldname)
if df and df.fieldtype == "Duration":
@ -487,6 +454,19 @@ def handle_duration_fieldtype_values(doctype, data, fields):
return data
def parse_field(field: str) -> tuple[str | None, str]:
"""Parse a field into parenttype and fieldname."""
key = field.split(" as ")[0]
if key.startswith(("count(", "sum(", "avg(")):
raise ValueError
if "." in key:
return key.split(".")[0][4:-1], key.split(".")[1].strip("`")
return None, key.strip("`")
@frappe.whitelist()
def delete_items():
"""delete selected items"""

31
frappe/desk/utils.py
View file

@ -27,3 +27,34 @@ def validate_route_conflict(doctype, name):
def slug(name):
return name.lower().replace(" ", "-")
def pop_csv_params(form_dict):
"""Pop csv params from form_dict and return them as a dict."""
from csv import QUOTE_NONNUMERIC
from frappe.utils.data import cint, cstr
return {
"delimiter": cstr(form_dict.pop("csv_delimiter", ","))[0],
"quoting": cint(form_dict.pop("csv_quoting", QUOTE_NONNUMERIC)),
}
def get_csv_bytes(data: list[list], csv_params: dict) -> bytes:
"""Convert data to csv bytes."""
from csv import writer
from io import StringIO
file = StringIO()
csv_writer = writer(file, **csv_params)
csv_writer.writerows(data)
return file.getvalue().encode("utf-8")
def provide_binary_file(filename: str, extension: str, content: bytes) -> None:
"""Provide a binary file to the client."""
frappe.response["type"] = "binary"
frappe.response["filecontent"] = content
frappe.response["filename"] = f"{filename}.{extension}"

13
frappe/email/doctype/email_queue/email_queue.py
View file

@ -26,6 +26,7 @@ from frappe.utils import (
cstr,
get_hook_method,
get_string_between,
get_url,
nowdate,
sbool,
split_emails,
@ -293,15 +294,11 @@ class SendMailContext:
message = self.include_attachments(message)
return message
def get_tracker_str(self):
tracker_url_html = '<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'
message = ""
def get_tracker_str(self) -> str:
if frappe.conf.use_ssl and self.email_account_doc.track_email_status:
message = quopri.encodestring(
tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
).decode()
return message
tracker_url_html = f'<img src="{get_url()}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={self.queue_doc.communication}"/>'
return quopri.encodestring(tracker_url_html.encode()).decode()
return ""
def get_unsubscribe_str(self, recipient_email: str) -> str:
unsubscribe_url = ""

1
frappe/hooks.py
View file

@ -220,7 +220,6 @@ scheduler_events = {
"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
"frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports",
"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",
"frappe.utils.subscription.enable_manage_subscription",
],

36
frappe/model/db_query.py
View file

@ -384,14 +384,21 @@ class DatabaseQuery:
_raise_exception()
for field in self.fields:
lower_field = field.lower().strip()
if SUB_QUERY_PATTERN.match(field):
if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords):
_raise_exception()
if lower_field[0] == "(":
subquery_token = lower_field[1:].lstrip().split(" ", 1)[0]
if subquery_token in blacklisted_keywords:
_raise_exception()
if any(f"{keyword}(" in field.lower() for keyword in blacklisted_functions):
_raise_exception()
function = lower_field.split("(", 1)[0].rstrip()
if function in blacklisted_functions:
frappe.throw(
_("Use of function {0} in field is restricted").format(function), exc=frappe.DataError
)
if "@" in field.lower():
if "@" in lower_field:
# prevent access to global variables
_raise_exception()
@ -407,7 +414,7 @@ class DatabaseQuery:
if STRICT_FIELD_PATTERN.match(field):
frappe.throw(_("Illegal SQL Query"))
if STRICT_UNION_PATTERN.match(field.lower()):
if STRICT_UNION_PATTERN.match(lower_field):
frappe.throw(_("Illegal SQL Query"))
def extract_tables(self):
@ -904,12 +911,16 @@ class DatabaseQuery:
if self.order_by:
args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}"
def validate_order_by_and_group_by(self, parameters):
def validate_order_by_and_group_by(self, parameters: str):
"""Check order by, group by so that atleast one column is selected and does not have subquery"""
if not parameters:
return
blacklisted_sql_functions = {
"sleep",
}
_lower = parameters.lower()
if "select" in _lower and "from" in _lower:
frappe.throw(_("Cannot use sub-query in order by"))
@ -917,13 +928,20 @@ class DatabaseQuery:
frappe.throw(_("Illegal SQL Query"))
for field in parameters.split(","):
if "." in field and field.strip().startswith("`tab"):
tbl = field.strip().split(".")[0]
field = field.strip()
function = field.split("(", 1)[0].rstrip().lower()
full_field_name = "." in field and field.startswith("`tab")
if full_field_name:
tbl = field.split(".", 1)[0]
if tbl not in self.tables:
if tbl.startswith("`"):
tbl = tbl[4:-1]
frappe.throw(_("Please select atleast 1 column from {0} to sort/group").format(tbl))
if function in blacklisted_sql_functions:
frappe.throw(_("Cannot use {0} in order/group by").format(field))
def add_limit(self):
if self.limit_page_length:
return f"limit {self.limit_page_length} offset {self.limit_start}"

38
frappe/model/document.py
View file

@ -3,6 +3,7 @@
import hashlib
import json
import time
from typing import Any, Generator, Iterable
from werkzeug.exceptions import NotFound
@ -1593,3 +1594,40 @@ def execute_action(__doctype, __name, __action, **kwargs):
doc.add_comment("Comment", _("Action Failed") + "<br><br>" + msg)
doc.notify_update()
def bulk_insert(
doctype: str,
documents: Iterable["Document"],
ignore_duplicates: bool = False,
chunk_size=10_000,
):
"""Insert simple Documents objects to database in bulk.
Warning/Info:
- All documents are inserted without triggering ANY hooks.
- This function assumes you've done the due dilligence and inserts in similar fashion as db_insert
- Documents can be any iterable / generator containing Document objects
"""
columns = frappe.get_meta(doctype).get_valid_columns()
values = _document_values_generator(documents, columns)
frappe.db.bulk_insert(
doctype, columns, values, ignore_duplicates=ignore_duplicates, chunk_size=chunk_size
)
def _document_values_generator(
documents: Iterable["Document"],
columns: list[str],
) -> Generator[tuple[Any], None, None]:
for doc in documents:
doc.creation = doc.modified = now()
doc.created_by = doc.modified_by = frappe.session.user
doc_values = doc.get_valid_dict(
convert_dates_to_str=True,
ignore_nulls=True,
ignore_virtual=True,
)
yield tuple(doc_values.get(col) for col in columns)

2
frappe/model/meta.py
View file

@ -230,7 +230,7 @@ class Meta(Document):
return fields
def get_valid_columns(self):
def get_valid_columns(self) -> list[str]:
if not hasattr(self, "_valid_columns"):
table_exists = frappe.db.table_exists(self.name)
if self.name in self.special_doctypes and table_exists:

2
frappe/modules/utils.py
View file

@ -296,7 +296,7 @@ def make_boilerplate(
controller_body = indent(
dedent(
"""
def db_insert(self):
def db_insert(self, *args, **kwargs):
pass
def load_from_db(self):

2
frappe/patches.txt
View file

@ -195,6 +195,8 @@ frappe.patches.v14_0.setup_likes_from_feedback
frappe.patches.v14_0.update_webforms
frappe.patches.v14_0.delete_payment_gateways
frappe.patches.v15_0.remove_event_streaming
frappe.patches.v15_0.remove_prepared_report_settings_from_system_settings
frappe.patches.v15_0.copy_disable_prepared_report_to_prepared_report
[post_model_sync]
frappe.patches.v14_0.drop_data_import_legacy

6
frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py Normal file
View file

@ -0,0 +1,6 @@
import frappe
def execute():
table = frappe.qb.DocType("Report")
frappe.qb.update(table).set(table.prepared_report, 0).where(table.disable_prepared_report == 1)

15
frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py Normal file
View file

@ -0,0 +1,15 @@
import frappe
from frappe.utils import cint
def execute():
expiry_period = (
cint(frappe.db.get_single_value("System Settings", "prepared_report_expiry_period")) or 30
)
frappe.get_single("Log Settings").register_doctype("Prepared Report", expiry_period)
singles = frappe.qb.DocType("Singles")
frappe.qb.from_(singles).delete().where(
(singles.doctype == "System Settings")
& (singles.field.isin(["enable_prepared_report_auto_deletion", "prepared_report_expiry_period"]))
).run()

14
frappe/public/js/frappe/form/controls/base_input.js
View file

@ -83,7 +83,9 @@ frappe.ui.form.ControlInput = class ControlInput extends frappe.ui.form.Control
me.value = me.doc[me.df.fieldname] || "";
}
if (me.can_write()) {
let is_fetch_from_read_only = me.df.fetch_from && !me.df.fetch_if_empty;
if (me.can_write() && !is_fetch_from_read_only) {
me.disp_area && $(me.disp_area).toggle(false);
$(me.input_area).toggle(true);
me.$input && me.$input.prop("disabled", false);
@ -101,6 +103,16 @@ frappe.ui.form.ControlInput = class ControlInput extends frappe.ui.form.Control
}
}
me.$input && me.$input.prop("disabled", true);
if (is_fetch_from_read_only) {
$(me.disp_area).attr(
"title",
__(
"This value is fetched from {0}'s {1} field",
me.df.fetch_from.split(".")
)
);
}
}
me.set_description();

97
frappe/public/js/frappe/form/grid.js
View file

@ -61,57 +61,56 @@ export default class Grid {
make() {
let template = `
<label class="control-label">${__(this.df.label || "")}</label>
<span class="ml-1 help"></span>
<p class="text-muted small grid-description"></p>
<div class="grid-custom-buttons grid-field"></div>
<div class="form-grid-container">
<div class="form-grid">
<div class="grid-heading-row"></div>
<div class="grid-body">
<div class="rows"></div>
<div class="grid-empty text-center">
<img
src="/assets/frappe/images/ui-states/grid-empty-state.svg"
alt="Grid Empty State"
class="grid-empty-illustration"
>
${__("No Data")}
<div class="grid-field">
<label class="control-label">${__(this.df.label || "")}</label>
<span class="ml-1 help"></span>
<p class="text-muted small grid-description"></p>
<div class="grid-custom-buttons"></div>
<div class="form-grid-container">
<div class="form-grid">
<div class="grid-heading-row"></div>
<div class="grid-body">
<div class="rows"></div>
<div class="grid-empty text-center">
<img
src="/assets/frappe/images/ui-states/grid-empty-state.svg"
alt="Grid Empty State"
class="grid-empty-illustration"
>
${__("No Data")}
</div>
</div>
</div>
</div>
</div>
<div class="small form-clickable-section grid-footer">
<div class="flex justify-between">
<div class="grid-buttons">
<button class="btn btn-xs btn-danger grid-remove-rows hidden"
style="margin-right: 4px;"
data-action="delete_rows">
${__("Delete")}
</button>
<button class="btn btn-xs btn-danger grid-remove-all-rows hidden"
style="margin-right: 4px;"
data-action="delete_all_rows">
${__("Delete All")}
</button>
<button class="grid-add-multiple-rows btn btn-xs btn-secondary hidden"
style="margin-right: 4px;">
${__("Add Multiple")}</a>
</button>
<!-- hack to allow firefox include this in tabs -->
<button class="btn btn-xs btn-secondary grid-add-row">
${__("Add Row")}
</button>
</div>
<div class="grid-pagination">
</div>
<div class="text-right">
<a href="#" class="grid-download btn btn-xs btn-secondary hidden">
${__("Download")}
</a>
<a href="#" class="grid-upload btn btn-xs btn-secondary hidden">
${__("Upload")}
</a>
<div class="small form-clickable-section grid-footer">
<div class="flex justify-between">
<div class="grid-buttons">
<button class="btn btn-xs btn-danger grid-remove-rows hidden"
data-action="delete_rows">
${__("Delete")}
</button>
<button class="btn btn-xs btn-danger grid-remove-all-rows hidden"
data-action="delete_all_rows">
${__("Delete All")}
</button>
<button class="grid-add-multiple-rows btn btn-xs btn-secondary hidden">
${__("Add Multiple")}</a>
</button>
<!-- hack to allow firefox include this in tabs -->
<button class="btn btn-xs btn-secondary grid-add-row">
${__("Add Row")}
</button>
</div>
<div class="grid-pagination">
</div>
<div class="grid-bulk-actions text-right">
<button class="grid-download btn btn-xs btn-secondary hidden">
${__("Download")}
</button>
<button class="grid-upload btn btn-xs btn-secondary hidden">
${__("Upload")}
</button>
</div>
</div>
</div>
</div>
@ -1150,7 +1149,7 @@ export default class Grid {
const $wrapper = position === "top" ? this.grid_custom_buttons : this.grid_buttons;
let $btn = this.custom_buttons[label];
if (!$btn) {
$btn = $(`<button class="btn btn-default btn-xs btn-custom">${__(label)}</button>`)
$btn = $(`<button class="btn btn-secondary btn-xs btn-custom">${__(label)}</button>`)
.prependTo($wrapper)
.on("click", click);
this.custom_buttons[label] = $btn;

4
frappe/public/js/frappe/form/linked_with.js
View file

@ -26,8 +26,8 @@ frappe.ui.form.LinkedWith = class LinkedWith {
this.dialog.on_page_show = () => {
frappe
.xcall("frappe.desk.form.linked_with.get", {
doctype: cur_frm.doctype,
docname: cur_frm.docname,
doctype: this.frm.doctype,
docname: this.frm.docname,
})
.then((r) => {
this.frm.__linked_docs = r;

4
frappe/public/js/frappe/views/kanban/kanban_view.js
View file

@ -175,9 +175,7 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView {
cur_list: this,
user_settings: this.view_user_settings,
});
}
if (this.kanban && board_name === this.kanban.board_name) {
} else if (board_name === this.kanban.board_name) {
this.kanban.update(this.data);
}
}

102
frappe/public/js/frappe/views/reports/query_report.js
View file

@ -866,8 +866,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
let filters = this.get_filter_values(true);
return new Promise((resolve) =>
frappe.call({
method: "frappe.desk.query_report.background_enqueue_run",
type: "GET",
method: "frappe.core.doctype.prepared_report.prepared_report.make_prepared_report",
args: {
report_name: this.report_name,
filters: filters,
@ -1412,70 +1411,55 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
return;
}
let export_dialog_fields = [
{
label: __("Select File Format"),
fieldname: "file_format",
fieldtype: "Select",
options: ["Excel", "CSV"],
default: "Excel",
reqd: 1,
},
];
let extra_fields = null;
if (this.tree_report) {
export_dialog_fields.push({
label: __("Include indentation"),
fieldname: "include_indentation",
fieldtype: "Check",
});
extra_fields = [
{
label: __("Include indentation"),
fieldname: "include_indentation",
fieldtype: "Check",
},
];
}
this.export_dialog = frappe.prompt(
export_dialog_fields,
({ file_format, include_indentation }) => {
this.export_dialog = frappe.report_utils.get_export_dialog(
__(this.report_name),
extra_fields,
({ file_format, include_indentation, csv_delimiter, csv_quoting }) => {
this.make_access_log("Export", file_format);
if (file_format === "CSV") {
const column_row = this.columns.reduce((acc, col) => {
if (!col.hidden) {
acc.push(__(col.label));
}
return acc;
}, []);
const data = this.get_data_for_csv(include_indentation);
const out = [column_row].concat(data);
frappe.tools.downloadify(out, null, this.report_name);
} else {
let filters = this.get_filter_values(true);
if (frappe.urllib.get_dict("prepared_report_name")) {
filters = Object.assign(
frappe.urllib.get_dict("prepared_report_name"),
filters
);
}
const visible_idx = this.datatable.bodyRenderer.visibleRowIndices;
if (visible_idx.length + 1 === this.data.length) {
visible_idx.push(visible_idx.length);
}
const args = {
cmd: "frappe.desk.query_report.export_query",
report_name: this.report_name,
custom_columns: this.custom_columns.length ? this.custom_columns : [],
file_format_type: file_format,
filters: filters,
visible_idx,
include_indentation,
};
open_url_post(frappe.request.url, args);
let filters = this.get_filter_values(true);
if (frappe.urllib.get_dict("prepared_report_name")) {
filters = Object.assign(
frappe.urllib.get_dict("prepared_report_name"),
filters
);
}
},
__("Export Report: {0}", [this.report_name]),
__("Download")
const visible_idx = this.datatable.bodyRenderer.visibleRowIndices;
if (visible_idx.length + 1 === this.data.length) {
visible_idx.push(visible_idx.length);
}
const args = {
cmd: "frappe.desk.query_report.export_query",
report_name: this.report_name,
custom_columns: this.custom_columns.length ? this.custom_columns : [],
file_format_type: file_format,
filters: filters,
visible_idx,
csv_delimiter,
csv_quoting,
include_indentation,
};
open_url_post(frappe.request.url, args);
this.export_dialog.hide();
}
);
this.export_dialog.show();
}
get_data_for_csv(include_indentation) {

141
frappe/public/js/frappe/views/reports/report_utils.js
View file

@ -158,4 +158,145 @@ frappe.report_utils = {
};
return get_result[fn](values);
},
get_export_dialog(report_name, extra_fields, callback) {
const fields = [
{
label: "File Format",
fieldname: "file_format",
fieldtype: "Select",
options: ["Excel", "CSV"],
default: "Excel",
reqd: 1,
},
{
fieldtype: "Section Break",
fieldname: "csv_settings",
label: "Settings",
collapsible: 1,
depends_on: "eval:doc.file_format=='CSV'",
},
{
fieldtype: "Data",
label: "CSV Delimiter",
fieldname: "csv_delimiter",
default: ",",
length: 1,
depends_on: "eval:doc.file_format=='CSV'",
},
{
fieldtype: "Select",
label: "CSV Quoting",
fieldname: "csv_quoting",
options: [
{ value: 0, label: "Minimal" },
{ value: 1, label: "All" },
{ value: 2, label: "Non-numeric" },
{ value: 3, label: "None" },
],
default: 2,
depends_on: "eval:doc.file_format=='CSV'",
},
{
fieldtype: "Small Text",
label: "CSV Preview",
fieldname: "csv_preview",
read_only: 1,
depends_on: "eval:doc.file_format=='CSV'",
},
];
if (extra_fields) {
fields.push(
{
fieldtype: "Section Break",
fieldname: "extra_fields",
collapsible: 0,
},
...extra_fields
);
}
const dialog = new frappe.ui.Dialog({
title: __("Export Report: {0}", [report_name], "Export report"),
fields: fields,
primary_action_label: __("Download", null, "Export report"),
primary_action: callback,
});
function update_csv_preview(dialog) {
const is_query_report = frappe.get_route()[0] === "query-report";
const report = is_query_report ? frappe.query_report : cur_list;
const columns = report.columns.filter((col) => col.hidden !== 1);
PREVIEW_DATA = [
columns.map((col) => __(is_query_report ? col.label : col.name)),
...report.data
.slice(0, 3)
.map((row) =>
columns.map((col) => row[is_query_report ? col.fieldname : col.field])
),
];
dialog.set_value(
"csv_preview",
frappe.report_utils.get_csv_preview(
PREVIEW_DATA,
dialog.get_value("csv_quoting"),
dialog.get_value("csv_delimiter")
)
);
}
dialog.fields_dict["file_format"].df.onchange = () => update_csv_preview(dialog);
dialog.fields_dict["csv_quoting"].df.onchange = () => update_csv_preview(dialog);
dialog.fields_dict["csv_delimiter"].df.onchange = () => update_csv_preview(dialog);
return dialog;
},
get_csv_preview(data, quoting, delimiter) {
// data: array of arrays
// quoting: 0 - minimal, 1 - all, 2 - non-numeric, 3 - none
// delimiter: any single character
quoting = cint(quoting);
const QUOTING = {
Minimal: 0,
All: 1,
NonNumeric: 2,
None: 3,
};
if (delimiter.length > 1) {
frappe.throw(__("Delimiter must be a single character"));
}
if (0 > quoting || quoting > 3) {
frappe.throw(__("Quoting must be between 0 and 3"));
}
return data
.map((row) => {
return row
.map((col) => {
if (typeof col == "string" && col.includes('"')) {
col = col.replace(/"/g, '""');
}
switch (quoting) {
case QUOTING.Minimal:
return typeof col === "string" && col.includes(delimiter)
? `"${col}"`
: `${col}`;
case QUOTING.All:
return `"${col}"`;
case QUOTING.NonNumeric:
return isNaN(col) ? `"${col}"` : `${col}`;
case QUOTING.None:
return `${col}`;
}
})
.join(delimiter);
})
.join("\n");
},
};

44
frappe/public/js/frappe/views/reports/report_view.js
View file

@ -1482,33 +1482,31 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
action: () => {
const args = this.get_args();
const selected_items = this.get_checked_items(true);
let fields = [
{
fieldtype: "Select",
label: __("Select File Type"),
fieldname: "file_format_type",
options: ["Excel", "CSV"],
default: "Excel",
},
];
if (this.total_count > this.count_without_children || args.page_length) {
fields.push({
fieldtype: "Check",
fieldname: "export_all_rows",
label: __("Export All {0} rows?", [(this.total_count + "").bold()]),
});
let extra_fields = null;
if (this.total_count > (this.count_without_children || args.page_length)) {
extra_fields = [
{
fieldtype: "Check",
fieldname: "export_all_rows",
label: __("Export All {0} rows?", [`<b>${this.total_count}</b>`]),
},
];
}
const d = new frappe.ui.Dialog({
title: __("Export Report: {0}", [__(this.doctype)]),
fields: fields,
primary_action_label: __("Download"),
primary_action: (data) => {
const d = frappe.report_utils.get_export_dialog(
__(this.doctype),
extra_fields,
(data) => {
args.cmd = "frappe.desk.reportview.export_query";
args.file_format_type = data.file_format_type;
args.file_format_type = data.file_format;
args.title = this.report_name || this.doctype;
if (data.file_format == "CSV") {
args.csv_delimiter = data.csv_delimiter;
args.csv_quoting = data.csv_quoting;
}
if (this.add_totals_row) {
args.add_totals_row = 1;
}
@ -1528,8 +1526,8 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
open_url_post(frappe.request.url, args);
d.hide();
},
});
}
);
d.show();
},

9
frappe/public/scss/common/grid.scss
View file

@ -398,16 +398,21 @@
}
}
.grid-buttons {
.grid-buttons, .grid-bulk-actions {
display: inline-flex;
}
.grid-footer {
.grid-footer, .grid-custom-buttons {
padding: var(--padding-sm) 0px;
background-color: var(--fg-color);
.btn {
box-shadow: none;
margin-top: -3px;
margin-bottom: -3px;
}
.btn:not(:last-child) {
margin-right: 4px;
}
}

77
frappe/tests/test_db_query.py
View file

@ -418,6 +418,43 @@ class TestReportview(FrappeTestCase):
)
self.assertTrue("date_diff" in data[0])
with self.assertRaises(frappe.DataError):
DatabaseQuery("DocType").execute(
fields=["name", "issingle", "if (issingle=1, (select name from tabUser), count(name))"],
limit_start=0,
limit_page_length=1,
)
with self.assertRaises(frappe.DataError):
DatabaseQuery("DocType").execute(
fields=["name", "issingle", "if(issingle=1, (select name from tabUser), count(name))"],
limit_start=0,
limit_page_length=1,
)
with self.assertRaises(frappe.DataError):
DatabaseQuery("DocType").execute(
fields=[
"name",
"issingle",
"( select name from `tabUser` where `tabDocType`.owner = `tabUser`.name )",
],
limit_start=0,
limit_page_length=1,
ignore_permissions=True,
)
with self.assertRaises(frappe.DataError):
DatabaseQuery("DocType").execute(
fields=[
"name",
"issingle",
"(select name from `tabUser` where `tabDocType`.owner = `tabUser`.name )",
],
limit_start=0,
limit_page_length=1,
)
def test_nested_permission(self):
frappe.set_user("Administrator")
create_nested_doctype()
@ -513,6 +550,46 @@ class TestReportview(FrappeTestCase):
)
self.assertTrue("DefaultValue" in [d["name"] for d in out])
def test_order_by_group_by_sanitizer(self):
# order by with blacklisted function
with self.assertRaises(frappe.ValidationError):
DatabaseQuery("DocType").execute(
fields=["name"],
order_by="sleep (1) asc",
)
# group by with blacklisted function
with self.assertRaises(frappe.ValidationError):
DatabaseQuery("DocType").execute(
fields=["name"],
group_by="SLEEP(0)",
)
# sub query in order by
with self.assertRaises(frappe.ValidationError):
DatabaseQuery("DocType").execute(
fields=["name"],
order_by="(select rank from tabRankedDocTypes where tabRankedDocTypes.name = tabDocType.name) asc",
)
# validate allowed usage
DatabaseQuery("DocType").execute(
fields=["name"],
order_by="name asc",
)
DatabaseQuery("DocType").execute(
fields=["name"],
order_by="name asc",
group_by="name",
)
# check mariadb specific syntax
if frappe.db.db_type == "mariadb":
DatabaseQuery("DocType").execute(
fields=["name"],
order_by="timestamp(modified)",
)
def test_of_not_of_descendant_ancestors(self):
frappe.set_user("Administrator")
clear_user_permissions_for_doctype("Nested DocType")

18
frappe/tests/test_document.py
View file

@ -469,3 +469,21 @@ class TestDocumentWebView(FrappeTestCase):
# Logged-in user can access the page without key
self.assertEqual(self.get(url_without_key, "Administrator").status, "200 OK")
def test_bulk_inserts(self):
from frappe.model.document import bulk_insert
doctype = "ToDo"
sent_todo = set()
def doc_generator():
for i in range(690):
doc = frappe.new_doc(doctype)
doc.name = doc.description = frappe.generate_hash()
sent_todo.add(doc.name)
yield doc
bulk_insert(doctype, doc_generator(), chunk_size=100)
all_todos = set(frappe.get_all("ToDo", pluck="name"))
self.assertEqual(sent_todo - all_todos, set(), "All docs should be inserted")

43
frappe/tests/test_query_report.py
View file

@ -3,7 +3,7 @@
import frappe
import frappe.utils
from frappe.desk.query_report import build_xlsx_data
from frappe.desk.query_report import build_xlsx_data, export_query
from frappe.tests.utils import FrappeTestCase
from frappe.utils.xlsxutils import make_xlsx
@ -69,3 +69,44 @@ class TestQueryReport(FrappeTestCase):
for row in xlsx_data:
# column_b should be 'str' even with composite cell value
self.assertEqual(type(row[1]), str)
def test_csv(self):
from csv import QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC, DictReader
from io import StringIO
REPORT_NAME = "Test CSV Report"
REF_DOCTYPE = "DocType"
REPORT_COLUMNS = ["name", "module", "issingle"]
if not frappe.db.exists("Report", REPORT_NAME):
report = frappe.new_doc("Report")
report.report_name = REPORT_NAME
report.ref_doctype = "User"
report.report_type = "Query Report"
report.query = frappe.qb.from_(REF_DOCTYPE).select(*REPORT_COLUMNS).limit(10).get_sql()
report.is_standard = "No"
report.save()
for delimiter in (",", ";", "\t", "|"):
for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC):
frappe.local.form_dict = frappe._dict(
{
"report_name": REPORT_NAME,
"file_format_type": "CSV",
"csv_quoting": quoting,
"csv_delimiter": delimiter,
"include_indentation": 0,
"visible_idx": [0, 1, 2],
}
)
export_query()
self.assertTrue(frappe.response["filename"].endswith(".csv"))
self.assertEqual(frappe.response["type"], "binary")
with StringIO(frappe.response["filecontent"].decode("utf-8")) as result:
reader = DictReader(result, delimiter=delimiter, quoting=quoting)
row = reader.__next__()
for column in REPORT_COLUMNS:
self.assertIn(column, row)
frappe.delete_doc("Report", REPORT_NAME, delete_permanently=True)

34
frappe/tests/test_reportview.py Normal file
View file

@ -0,0 +1,34 @@
# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import frappe
from frappe.desk.reportview import export_query
from frappe.tests.utils import FrappeTestCase
class TestReportview(FrappeTestCase):
def test_csv(self):
from csv import QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC, DictReader
from io import StringIO
frappe.local.form_dict = frappe._dict(
doctype="DocType",
file_format_type="CSV",
fields=("name", "module", "issingle"),
filters={"issingle": 1, "module": "Core"},
)
for delimiter in (",", ";", "\t", "|"):
frappe.local.form_dict.csv_delimiter = delimiter
for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC):
frappe.local.form_dict.csv_quoting = quoting
export_query()
self.assertTrue(frappe.response["filename"].endswith(".csv"))
self.assertEqual(frappe.response["type"], "binary")
with StringIO(frappe.response["filecontent"].decode("utf-8")) as result:
reader = DictReader(result, delimiter=delimiter, quoting=quoting)
for row in reader:
self.assertEqual(int(row["Is Single"]), 1)
self.assertEqual(row["Module"], "Core")

9
frappe/translations/de.csv
View file

@ -158,7 +158,7 @@ No Data,Keine Daten,
No address added yet.,Noch keine Adresse hinzugefügt.,
No contacts added yet.,Noch keine Kontakte hinzugefügt.,
No items found.,Keine Elemente gefunden.,
None,Keiner,
None,Keine,
Not Permitted,Nicht zulässig,
Not active,Nicht aktiv,
Notes,Hinweise,
@ -2202,7 +2202,6 @@ Select Columns,Spalten auswählen,
Select Document Type,Dokumenttyp auswählen,
Select Document Type or Role to start.,"Dokumententyp oder Rolle auswählen, um zu beginnen.",
Select Document Types to set which User Permissions are used to limit access.,"Dokumentenarten auswählen, um die Benutzerrechte, die den Zugriff einschränken, anzuwenden",
Select File Format,Wählen Sie Dateiformat,
Select File Type,Dateityp auswählen,
Select Language...,Sprache auswählen...,
Select Languages,Sprachenauswahl,
@ -4823,3 +4822,9 @@ Preview Chart,Vorschau erzeugen,
Please select X and Y fields,Bitte Felder für die X- und Y-Achse wählen,
Notification sent to,Benachrichtigung gesendet an,
Add to this activity by mailing to {0},"Senden Sie eine E-Mail an {0}, damit sie hier erscheint",
File Format,Dateiformat,
CSV Delimiter,Trennzeichen,
CSV Quoting,Anführungszeichen,
CSV Preview,Vorschau,
Non-numeric,Nicht-numerische,
Minimal,Minimal,

1 A4 A4
158 No address added yet. Noch keine Adresse hinzugefügt.
159 No contacts added yet. Noch keine Kontakte hinzugefügt.
160 No items found. Keine Elemente gefunden.
161 None Keiner Keine
162 Not Permitted Nicht zulässig
163 Not active Nicht aktiv
164 Notes Hinweise
2202 Select Document Type Dokumenttyp auswählen
2203 Select Document Type or Role to start. Dokumententyp oder Rolle auswählen, um zu beginnen.
2204 Select Document Types to set which User Permissions are used to limit access. Dokumentenarten auswählen, um die Benutzerrechte, die den Zugriff einschränken, anzuwenden
Select File Format Wählen Sie Dateiformat
2205 Select File Type Dateityp auswählen
2206 Select Language... Sprache auswählen...
2207 Select Languages Sprachenauswahl
4822 Please select X and Y fields Bitte Felder für die X- und Y-Achse wählen
4823 Notification sent to Benachrichtigung gesendet an
4824 Add to this activity by mailing to {0} Senden Sie eine E-Mail an {0}, damit sie hier erscheint
4825 File Format Dateiformat
4826 CSV Delimiter Trennzeichen
4827 CSV Quoting Anführungszeichen
4828 CSV Preview Vorschau
4829 Non-numeric Nicht-numerische
4830 Minimal Minimal

28
frappe/utils/background_jobs.py
View file

@ -3,13 +3,14 @@ import socket
import time
from collections import defaultdict
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Union
from typing import TYPE_CHECKING, Any, Literal, NoReturn, Union
from uuid import uuid4
import redis
from redis.exceptions import BusyLoadingError, ConnectionError
from rq import Connection, Queue, Worker
from rq.logutils import setup_loghandlers
from rq.worker import RandomWorker, RoundRobinWorker
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
import frappe
@ -34,9 +35,11 @@ def get_queues_timeout():
custom_workers_config = common_site_config.get("workers", {})
default_timeout = 300
# Note: Order matters here
# If no queues are specified then RQ prioritizes queues in specified order
return {
"default": default_timeout,
"short": default_timeout,
"default": default_timeout,
"long": 1500,
**{
worker: config.get("timeout", default_timeout)
@ -209,22 +212,37 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True,
frappe.destroy()
def start_worker(queue=None, quiet=False, rq_username=None, rq_password=None):
def start_worker(
queue: str | None = None,
quiet: bool = False,
rq_username: str | None = None,
rq_password: str | None = None,
burst: bool = False,
strategy: Literal["round_robin", "random"] | None = None,
) -> NoReturn | None:
"""Wrapper to start rq worker. Connects to redis and monitors these queues."""
DEQUEUE_STRATEGIES = {"round_robin": RoundRobinWorker, "random": RandomWorker}
with frappe.init_site():
# empty init is required to get redis_queue from common_site_config.json
redis_connection = get_redis_conn(username=rq_username, password=rq_password)
if queue:
queue = [q.strip() for q in queue.split(",")]
queues = get_queue_list(queue, build_queue_name=True)
queue_name = queue and generate_qname(queue)
if os.environ.get("CI"):
setup_loghandlers("ERROR")
WorkerKlass = DEQUEUE_STRATEGIES.get(strategy, Worker)
with Connection(redis_connection):
logging_level = "INFO"
if quiet:
logging_level = "WARNING"
Worker(queues, name=get_worker_name(queue_name)).work(logging_level=logging_level)
worker = WorkerKlass(queues, name=get_worker_name(queue_name))
worker.work(logging_level=logging_level, burst=burst)
def get_worker_name(queue):
@ -367,6 +385,8 @@ def generate_qname(qtype: str) -> str:
qnames are useful to define namespaces of customers.
"""
if isinstance(qtype, list):
qtype = ",".join(qtype)
return f"{get_bench_id()}:{qtype}"

2
frappe/utils/safe_exec.py
View file

@ -106,7 +106,6 @@ def get_safe_globals():
as_json=frappe.as_json,
dict=dict,
log=frappe.log,
_dict=frappe._dict,
args=form_dict,
frappe=NamespaceDict(
call=call_whitelisted_function,
@ -117,6 +116,7 @@ def get_safe_globals():
time_format=time_format,
format_date=frappe.utils.data.global_date_format,
form_dict=form_dict,
as_dict=frappe._dict,
bold=frappe.bold,
copy_doc=frappe.copy_doc,
errprint=frappe.errprint,

5
frappe/utils/weasyprint.py
View file

@ -4,6 +4,7 @@
import click
import frappe
from frappe import _
@frappe.whitelist()
@ -49,7 +50,11 @@ class PrintFormatGenerator:
self.base_url = frappe.utils.get_url()
self.print_format = frappe.get_doc("Print Format", print_format)
self.doc = doc
if letterhead == _("No Letterhead"):
letterhead = None
self.letterhead = frappe.get_doc("Letter Head", letterhead) if letterhead else None
self.build_context()
self.layout = self.get_layout(self.print_format)
self.context.layout = self.layout

8
frappe/utils/xlsxutils.py
View file

@ -108,8 +108,6 @@ def read_xls_file_from_attached_file(content):
def build_xlsx_response(data, filename):
xlsx_file = make_xlsx(data, filename)
# write out response as a xlsx type
frappe.response["filename"] = filename + ".xlsx"
frappe.response["filecontent"] = xlsx_file.getvalue()
frappe.response["type"] = "binary"
from frappe.desk.utils import provide_binary_file
provide_binary_file(filename, "xlsx", make_xlsx(data, filename).getvalue())

4
frappe/www/printview.py
View file

@ -335,8 +335,8 @@ def validate_print_permission(doc):
if frappe.has_permission(doc.doctype, ptype, doc) or frappe.has_website_permission(doc):
return
key = frappe.form_dict.get("key")
if key:
key = frappe.form_dict.key
if key and isinstance(key, str):
validate_key(key, doc)
else:
raise frappe.PermissionError(_("You do not have permission to view this document"))