From 5b9f85f1c469d169b287914421b588de93481d83 Mon Sep 17 00:00:00 2001 From: David Arnold Date: Wed, 18 Oct 2023 01:35:40 +0200 Subject: [PATCH 001/237] fix(geolocation): modal and state flow closes: #22796 --- .../js/frappe/form/controls/geolocation.js | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js index 56a8d2a073..827a9be315 100644 --- a/frappe/public/js/frappe/form/controls/geolocation.js +++ b/frappe/public/js/frappe/form/controls/geolocation.js @@ -13,30 +13,41 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f if (!this.disp_area) { return; } + if (!this.map_id) { + this.map_id = frappe.dom.get_unique_id(); + this.map_area = $( + `
+
+
` + ); - this.map_id = frappe.dom.get_unique_id(); - this.map_area = $( - `
-
-
` - ); + $(this.disp_area).html(this.map_area); + } - $(this.disp_area).html(this.map_area); + // show again on idempotent invocations $(this.disp_area).removeClass("like-disabled-input"); $(this.disp_area).css("display", "block"); if (this.frm) { - this.make_map(value); + this.make_map(); + if (value) { + this.bind_leaflet_data(value); + } } else { $(document).on("frappe.ui.Dialog:shown", () => { this.make_map(); + if (value) { + this.bind_leaflet_data(value); + } }); } } make_map(value) { - this.customize_draw_controls(); - this.bind_leaflet_map(); + if (!this.map) { + this.customize_draw_controls(); + this.bind_leaflet_map(); + } if (this.disabled) { this.map.dragging.disable(); this.map.touchZoom.disable(); @@ -47,17 +58,17 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f this.map.zoomControl.remove(); } else { this.bind_leaflet_draw_control(); - this.bind_leaflet_event_listeners(); - this.bind_leaflet_locate_control(); - this.bind_leaflet_data(value); + if (!this.bound_event_listeners) { + this.bind_leaflet_event_listeners(); + } + if (!this.locate_control) { + this.bind_leaflet_locate_control(); + } } } bind_leaflet_data(value) { /* render raw value from db into map */ - if (!this.map || !value) { - return; - } this.clear_editable_layers(); const data_layers = new L.FeatureGroup().addLayer( @@ -159,14 +170,14 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f } bind_leaflet_draw_control() { - if ( - !frappe.perm.has_perm(this.doctype, this.df.permlevel, "write", this.doc) || - this.df.read_only - ) { - return; + if (!this.draw_control) { + this.draw_control = this.get_leaflet_controls(); + } + if (this.disp_status == "Write") { + this.draw_control.addTo(this.map); + } else { + this.draw_control.remove(); } - - this.map.addControl(this.get_leaflet_controls()); } get_leaflet_controls() { @@ -204,6 +215,7 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f } bind_leaflet_event_listeners() { + this.bound_event_listeners = true; this.map.on("draw:created", (e) => { var type = e.layerType, layer = e.layer; From e059aa385fb2bfa005add29aa5d9fe10f55f55dd Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Thu, 9 Nov 2023 12:08:05 +0530 Subject: [PATCH 002/237] feat: add check to include filters in popup --- .../js/frappe/views/reports/query_report.js | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index b92c3166f5..e03f227055 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1475,21 +1475,34 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { return; } - let extra_fields = null; + let extra_fields = []; + if (this.tree_report) { - extra_fields = [ - { - label: __("Include indentation"), - fieldname: "include_indentation", - fieldtype: "Check", - }, - ]; + extra_fields.push({ + label: __("Include indentation"), + fieldname: "include_indentation", + fieldtype: "Check", + }); + } + + if (this.filters.length > 0) { + extra_fields.push({ + label: __("Include filters"), + fieldname: "include_filters", + fieldtype: "Check", + }); } this.export_dialog = frappe.report_utils.get_export_dialog( __(this.report_name), extra_fields, - ({ file_format, include_indentation, csv_delimiter, csv_quoting }) => { + ({ + file_format, + include_indentation, + include_filters, + csv_delimiter, + csv_quoting, + }) => { this.make_access_log("Export", file_format); let filters = this.get_filter_values(true); @@ -1515,6 +1528,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { csv_delimiter, csv_quoting, include_indentation, + include_filters, }; open_url_post(frappe.request.url, args); From a52d1870dc8e803f28c1410860ec5b54a9560c69 Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Thu, 9 Nov 2023 12:09:42 +0530 Subject: [PATCH 003/237] feat: add filter values while building report xlsx data --- frappe/desk/query_report.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 7ca483d806..0a25dd3b8f 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -318,6 +318,7 @@ def export_query(): file_format_type = form_params.file_format_type custom_columns = frappe.parse_json(form_params.custom_columns or "[]") include_indentation = form_params.include_indentation + include_filters = form_params.include_filters visible_idx = form_params.visible_idx if isinstance(visible_idx, str): @@ -327,6 +328,7 @@ def export_query(): report_name, form_params.filters, custom_columns=custom_columns, are_default_filters=False ) data = frappe._dict(data) + data.filters = form_params.filters if not data.columns: frappe.respond_as_web_page( _("No data to export"), @@ -335,7 +337,9 @@ def export_query(): return format_duration_fields(data) - xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation) + xlsx_data, column_widths = build_xlsx_data( + data, visible_idx, include_indentation, include_filters=include_filters + ) if file_format_type == "CSV": content = get_csv_bytes(xlsx_data, csv_params) @@ -360,7 +364,9 @@ def format_duration_fields(data: frappe._dict) -> None: row[index] = format_duration(row[index]) -def build_xlsx_data(data, visible_idx, include_indentation, ignore_visible_idx=False): +def build_xlsx_data( + data, visible_idx, include_indentation, include_filters=False, ignore_visible_idx=False +): EXCEL_TYPES = ( str, bool, @@ -380,17 +386,30 @@ def build_xlsx_data(data, visible_idx, include_indentation, ignore_visible_idx=F # Note: converted for faster lookups visible_idx = set(visible_idx) - result = [[]] + result = [] column_widths = [] + if cint(include_filters): + filter_data = [] + filters = data.filters + for filter_name, filter_value in filters.items(): + if filter_value in ["", None, []]: + continue + filter_value = ", ".join(filter_value) if isinstance(filter_value, list) else cstr(filter_value) + filter_data.append([cstr(filter_name) + ": ", filter_value]) + filter_data.append([]) + result += filter_data + + column_data = [] for column in data.columns: if column.get("hidden"): continue - result[0].append(_(column.get("label"))) + column_data.append(_(column.get("label"))) column_width = cint(column.get("width", 0)) # to convert into scale accepted by openpyxl column_width /= 10 column_widths.append(column_width) + result.append(column_data) # build table from result for row_idx, row in enumerate(data.result): From 3f148a0f7edb6242e5dda962317e577575f8d9c2 Mon Sep 17 00:00:00 2001 From: David Arnold Date: Sat, 18 Nov 2023 16:10:33 +0100 Subject: [PATCH 004/237] fix(list_settings): don't count tags to total fields --- frappe/public/js/frappe/list/list_view.js | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 893c212f23..e865636e03 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -358,11 +358,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { }); } - this.columns.push({ - type: "Tag", - }); - - // 2nd column: Status indicator + // 3nd column: Status indicator if (frappe.has_indicator(this.doctype)) { // indicator this.columns.push({ @@ -407,11 +403,12 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { this.columns = this.columns.slice(0, this.list_view_settings.total_fields || total_fields); - if ( - !this.settings.hide_name_column && - this.meta.title_field && - this.meta.title_field !== "name" - ) { + // 2st column: tag - normally hidden doesn't count towards total_fields + this.columns.splice(1, 0, { + type: "Tag", + }); + + if (!this.settings.hide_name_column) { this.columns.push({ type: "Field", df: { @@ -426,10 +423,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { let fields_order = []; let fields = JSON.parse(this.list_view_settings.fields); - //title and tags field is fixed + // title field is fixed fields_order.push(this.columns[0]); - fields_order.push(this.columns[1]); - this.columns.splice(0, 2); + this.columns.splice(0, 1); for (let fld in fields) { for (let col in this.columns) { From 4203c3b13da0821419f0f513121ba9926da7cd00 Mon Sep 17 00:00:00 2001 From: HarryPaulo Date: Mon, 20 Nov 2023 22:31:12 -0300 Subject: [PATCH 005/237] fix: load languages that have capital letters --- frappe/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/translate.py b/frappe/translate.py index cc0771583e..2f3341fd35 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -200,7 +200,7 @@ def get_translations_from_apps(lang, apps=None): translations = {} for app in apps or frappe.get_installed_apps(_ensure_on_bench=True): - path = frappe.get_app_path(app, "translations", lang + ".csv") + path = os.path.join(frappe.get_app_path(app, "translations"), lang + ".csv") translations.update(get_translation_dict_from_file(path, lang, app) or {}) if "-" in lang: parent = lang.split("-", 1)[0] From d258222f284ce3be474c765e3654682afcbab0a4 Mon Sep 17 00:00:00 2001 From: Corentin Flr <10946971+cogk@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:01:33 +0100 Subject: [PATCH 006/237] fix: Inherit text-align for .btn-reset --- frappe/public/scss/common/buttons.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/scss/common/buttons.scss b/frappe/public/scss/common/buttons.scss index 672498ada3..d0eb6da859 100644 --- a/frappe/public/scss/common/buttons.scss +++ b/frappe/public/scss/common/buttons.scss @@ -118,6 +118,7 @@ border: 0; font-size: inherit; background-color: inherit; + text-align: inherit; } [data-theme="dark"] { From f825acf92249cb585478bd8ef608abc173e6486c Mon Sep 17 00:00:00 2001 From: Corentin Flr <10946971+cogk@users.noreply.github.com> Date: Mon, 20 Nov 2023 22:14:29 +0100 Subject: [PATCH 007/237] fix(a11y): Make navbar more accessible * Use From b0c2f56daec819950d0fc57f8ba3f0edad41ff4f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 5 Dec 2023 21:11:35 +0530 Subject: [PATCH 050/237] Revert "feat: wkhtmltopdf logging (#19935)" This reverts commit 6354a018de798c49c69de28c3493b04fa43bf96c. --- frappe/utils/logger.py | 28 ---------------------------- frappe/utils/pdf.py | 12 ++---------- 2 files changed, 2 insertions(+), 38 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index e80973d9f5..360c5eaedc 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -1,8 +1,6 @@ # imports - standard imports import logging import os -import sys -from contextlib import contextmanager from copy import deepcopy from logging.handlers import RotatingFileHandler from typing import Literal @@ -126,29 +124,3 @@ def sanitized_dict(form_dict): if secret_kw in k: sanitized_dict[k] = "********" return sanitized_dict - - -@contextmanager -def pipe_to_log(logger_fn, stream=None): - "Pass an existing logger function e.g. logger.info. Stream defaults to stdout" - # late bind source - if stream is None: - stream = sys.stdout - - stream_int = stream.fileno() - r_int, w_int = os.pipe() - - # copy stream_fd before it is overwritten - with os.fdopen(os.dup(stream_int), "wb") as copied: - stream.flush() - os.dup2(w_int, stream_int) # $ exec >&pipe - try: - with os.fdopen(w_int, "wb"): - yield stream - finally: - # restore stream to its previous value - stream.flush() - os.dup2(copied.fileno(), stream_int) # $ exec >&copied - with os.fdopen(r_int, newline="") as r: - text = r.read() - logger_fn(text) diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index ecd87a6a0e..24b6896620 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -15,7 +15,6 @@ import frappe from frappe import _ from frappe.utils import scrub_urls from frappe.utils.jinja_globals import bundled_asset, is_rtl -from frappe.utils.logger import pipe_to_log PDF_CONTENT_ERRORS = [ "ContentNotFoundError", @@ -24,9 +23,6 @@ PDF_CONTENT_ERRORS = [ "RemoteHostClosedError", ] -logger = frappe.logger("wkhtmltopdf", max_size=100000, file_count=3) -logger.setLevel("INFO") - def pdf_header_html(soup, head, content, styles, html_id, css): return frappe.render_template( @@ -87,13 +83,8 @@ def get_pdf(html, options=None, output: PdfWriter | None = None): options.update({"disable-smart-shrinking": ""}) try: - # wkhtmltopdf writes the pdf to stdout and errors to stderr - # pdfkit v1.0.0 writes the pdf to file or returns it - # stderr is written to sys.stdout if verbose=True is supplied # Set filename property to false, so no file is actually created - # defaults to redirecting stdout - with pipe_to_log(logger.info): - filedata = pdfkit.from_string(html, False, options=options or {}, verbose=True) + filedata = pdfkit.from_string(html, options=options or {}, verbose=True) # create in-memory binary streams from filedata and create a PdfReader object reader = PdfReader(io.BytesIO(filedata)) @@ -151,6 +142,7 @@ def prepare_options(html, options): "print-media-type": None, "background": None, "images": None, + "quiet": None, # 'no-outline': None, "encoding": "UTF-8", # 'load-error-handling': 'ignore' From 1b328854ccd2dbcd0c51fd4ef401c6fcd3cdac89 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Dec 2023 18:00:23 +0100 Subject: [PATCH 051/237] fix(Report View): clear checked items --- frappe/public/js/frappe/views/reports/report_view.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index 1eb20646c9..fd425ebc30 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -1254,6 +1254,10 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { return items; } + clear_checked_items() { + this.datatable.rowmanager.checkAll(false); + } + save_report(save_type) { const _save_report = (name) => { // callback From cb3205e5698802cfefc9bbc7721cf07cff59d5c9 Mon Sep 17 00:00:00 2001 From: Maharshi Patel Date: Tue, 5 Dec 2023 23:31:58 +0530 Subject: [PATCH 052/237] fix: only update primary action if it is new action is provided Currently, the primary action is updated even if the new action is None. This breaks some old code that relies on the primary action being None when no action is provided. (used to update primary action label) I changed the code to only update the primary action if the new action function is provided --- frappe/public/js/frappe/ui/dialog.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/ui/dialog.js b/frappe/public/js/frappe/ui/dialog.js index b2ac9a7769..515b693482 100644 --- a/frappe/public/js/frappe/ui/dialog.js +++ b/frappe/public/js/frappe/ui/dialog.js @@ -180,11 +180,9 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.FieldGroup { this.footer.removeClass("hide"); this.has_primary_action = true; var me = this; - return this.get_primary_btn() - .removeClass("hide") - .html(label) - .off("click") - .on("click", function () { + const primary_btn = this.get_primary_btn().removeClass("hide").html(label); + if (typeof click == "function") { + primary_btn.off("click").on("click", function () { me.primary_action_fulfilled = true; // get values and send it // as first parameter to click callback @@ -193,6 +191,8 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.FieldGroup { if (!values) return; click && click.apply(me, [values]); }); + } + return primary_btn; } set_secondary_action(click) { From a9c4894ccfef40f404ca62c253f13ec7f790a1b0 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:16:27 +0100 Subject: [PATCH 053/237] fix: task_id parameter for publish_progress --- frappe/realtime.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/frappe/realtime.py b/frappe/realtime.py index b795df3782..fe62571c05 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -9,13 +9,16 @@ import frappe from frappe.utils.data import cstr -def publish_progress(percent, title=None, doctype=None, docname=None, description=None): +def publish_progress( + percent, title=None, doctype=None, docname=None, description=None, task_id=None +): publish_realtime( "progress", {"percent": percent, "title": title, "description": description}, user=None if doctype and docname else frappe.session.user, doctype=doctype, docname=docname, + task_id=task_id, ) @@ -41,8 +44,11 @@ def publish_realtime( if message is None: message = {} + if not task_id and hasattr(frappe.local, "task_id"): + task_id = frappe.local.task_id + if event is None: - event = "task_progress" if frappe.local.task_id else "global" + event = "task_progress" if task_id else "global" elif event == "msgprint" and not user: user = frappe.session.user elif event == "list_update": @@ -51,9 +57,6 @@ def publish_realtime( elif event == "docinfo_update": room = get_doc_room(doctype, docname) - if not task_id and hasattr(frappe.local, "task_id"): - task_id = frappe.local.task_id - if not room: if task_id: after_commit = False From 105c4a20fb4d5e2546dc8ea1fc22eb72a96c5295 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:19:56 +0100 Subject: [PATCH 054/237] refactor: better variable names --- frappe/desk/doctype/bulk_update/bulk_update.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 27ffb4ffb8..8a8583ab7a 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -74,8 +74,8 @@ def _bulk_action(doctype, docnames, action, data): failed = [] - for i, d in enumerate(docnames, 1): - doc = frappe.get_doc(doctype, d) + for idx, docname in enumerate(docnames, 1): + doc = frappe.get_doc(doctype, docname) try: message = "" if action == "submit" and doc.docstatus.is_draft(): @@ -93,12 +93,12 @@ def _bulk_action(doctype, docnames, action, data): doc.save() message = _("Updating {0}").format(doctype) else: - failed.append(d) + failed.append(docname) frappe.db.commit() - show_progress(docnames, message, i, d) + show_progress(docnames, message, idx, docname) except Exception: - failed.append(d) + failed.append(docname) frappe.db.rollback() return failed From 09395420b855ecfa8f77138ea03303a35248a6f3 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:30:13 +0100 Subject: [PATCH 055/237] feat: task_id for submit_cancel_or_update_docs --- .../desk/doctype/bulk_update/bulk_update.py | 16 ++++++++--- .../public/js/frappe/list/bulk_operations.js | 28 +++++++++---------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 8a8583ab7a..9ec7e44ae8 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -24,6 +24,7 @@ class BulkUpdate(Document): limit: DF.Int update_value: DF.SmallText # end: auto-generated types + @frappe.whitelist() def bulk_update(self): self.check_permission("write") @@ -45,12 +46,12 @@ class BulkUpdate(Document): @frappe.whitelist() -def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): +def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None, task_id=None): if isinstance(docnames, str): docnames = frappe.parse_json(docnames) if len(docnames) < 20: - return _bulk_action(doctype, docnames, action, data) + return _bulk_action(doctype, docnames, action, data, task_id) elif len(docnames) <= 500: frappe.msgprint(_("Bulk operation is enqueued in background."), alert=True) frappe.enqueue( @@ -59,6 +60,7 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): docnames=docnames, action=action, data=data, + task_id=task_id, queue="short", timeout=1000, ) @@ -68,11 +70,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): ) -def _bulk_action(doctype, docnames, action, data): +def _bulk_action(doctype, docnames, action, data, task_id=None): if data: data = frappe.parse_json(data) failed = [] + num_documents = len(docnames) for idx, docname in enumerate(docnames, 1): doc = frappe.get_doc(doctype, docname) @@ -95,7 +98,12 @@ def _bulk_action(doctype, docnames, action, data): else: failed.append(docname) frappe.db.commit() - show_progress(docnames, message, idx, docname) + frappe.publish_progress( + percent=idx / num_documents * 100, + title=message, + description=docname, + task_id=task_id, + ) except Exception: failed.append(docname) diff --git a/frappe/public/js/frappe/list/bulk_operations.js b/frappe/public/js/frappe/list/bulk_operations.js index a0271967b4..863c0abb90 100644 --- a/frappe/public/js/frappe/list/bulk_operations.js +++ b/frappe/public/js/frappe/list/bulk_operations.js @@ -209,28 +209,28 @@ export default class BulkOperations { submit_or_cancel(docnames, action = "submit", done = null) { action = action.toLowerCase(); - frappe - .call({ - method: "frappe.desk.doctype.bulk_update.bulk_update.submit_cancel_or_update_docs", - args: { - doctype: this.doctype, - action: action, - docnames: docnames, - }, + const task_id = Math.random().toString(36).slice(-5); + frappe.realtime.task_subscribe(task_id); + return frappe + .xcall("frappe.desk.doctype.bulk_update.bulk_update.submit_cancel_or_update_docs", { + doctype: this.doctype, + action: action, + docnames: docnames, + task_id: task_id, }) - .then((r) => { - let failed = r.message; - if (!failed) failed = []; - - if (failed.length && !r._server_messages) { + .then((failed) => { + if (failed?.length) { frappe.throw( __("Cannot {0} {1}", [action, failed.map((f) => f.bold()).join(", ")]) ); } - if (failed.length < docnames.length) { + if (failed?.length < docnames.length) { frappe.utils.play_sound(action); if (done) done(); } + }) + .finally(() => { + frappe.realtime.task_unsubscribe(task_id); }); } From 9ada76df6c4ed7ba0d5545f2ff95f69f11ef9739 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:30:43 +0100 Subject: [PATCH 056/237] chore: deprecate old show_progress --- frappe/desk/doctype/bulk_update/bulk_update.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 9ec7e44ae8..a0f5a45326 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -6,6 +6,7 @@ from frappe import _ from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.model.document import Document from frappe.utils import cint +from frappe.utils.deprecations import deprecated from frappe.utils.scheduler import is_scheduler_inactive @@ -112,6 +113,7 @@ def _bulk_action(doctype, docnames, action, data, task_id=None): return failed +@deprecated def show_progress(docnames, message, i, description): n = len(docnames) frappe.publish_progress(float(i) * 100 / n, title=message, description=description) From 54a83892bbb3d2d7cb546729678ee3e4465f283a Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:37:02 +0100 Subject: [PATCH 057/237] fix: better error message --- frappe/public/js/frappe/list/bulk_operations.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/list/bulk_operations.js b/frappe/public/js/frappe/list/bulk_operations.js index 863c0abb90..41fed778db 100644 --- a/frappe/public/js/frappe/list/bulk_operations.js +++ b/frappe/public/js/frappe/list/bulk_operations.js @@ -220,9 +220,17 @@ export default class BulkOperations { }) .then((failed) => { if (failed?.length) { - frappe.throw( - __("Cannot {0} {1}", [action, failed.map((f) => f.bold()).join(", ")]) - ); + const comma_separated_records = frappe.utils.comma_and(failed); + switch (action) { + case "submit": + frappe.throw(__("Cannot submit {0}.", [comma_separated_records])); + break; + case "cancel": + frappe.throw(__("Cannot cancel {0}.", [comma_separated_records])); + break; + default: + frappe.throw(__("Cannot {0} {1}.", [action, comma_separated_records])); + } } if (failed?.length < docnames.length) { frappe.utils.play_sound(action); From 68c0e6f85f36be71928593a2fbaf756115f22462 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:39:47 +0100 Subject: [PATCH 058/237] refactor: better parameter name --- frappe/public/js/frappe/list/bulk_operations.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/list/bulk_operations.js b/frappe/public/js/frappe/list/bulk_operations.js index 41fed778db..22cf51166d 100644 --- a/frappe/public/js/frappe/list/bulk_operations.js +++ b/frappe/public/js/frappe/list/bulk_operations.js @@ -218,9 +218,9 @@ export default class BulkOperations { docnames: docnames, task_id: task_id, }) - .then((failed) => { - if (failed?.length) { - const comma_separated_records = frappe.utils.comma_and(failed); + .then((failed_docnames) => { + if (failed_docnames?.length) { + const comma_separated_records = frappe.utils.comma_and(failed_docnames); switch (action) { case "submit": frappe.throw(__("Cannot submit {0}.", [comma_separated_records])); @@ -232,7 +232,7 @@ export default class BulkOperations { frappe.throw(__("Cannot {0} {1}.", [action, comma_separated_records])); } } - if (failed?.length < docnames.length) { + if (failed_docnames?.length < docnames.length) { frappe.utils.play_sound(action); if (done) done(); } From 8a2b9407c8f5bba07661f7d3bb0ebd5849722c8a Mon Sep 17 00:00:00 2001 From: ajiragroup <108009061+ajiragroup@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:36:41 +0545 Subject: [PATCH 059/237] fix: Modified Number system for Nepal (#23613) * Update utils.js Further refined number system for Nepal. Numbering in Nepal does not continue as 100 Crores/1000 Crores as in India, but it proceeds as: -> 1 Arba for 100 Crore -> 1 Kharba for 100 Arba (10,000 Cr) * Update number_systems.js Added number system for Nepal. Numbering in Nepal does not continue as 100 Crores/1000 Crores as in India, but it proceeds as: -> 1 Arba for 100 Crore -> 1 Kharba for 100 Arba (10,000 Cr) * chore: simpler condition * style: format --------- Co-authored-by: Ankush Menat Co-authored-by: Ankush Menat --- .../public/js/frappe/utils/number_systems.js | 22 +++++++++++++++++++ frappe/public/js/frappe/utils/utils.js | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/number_systems.js b/frappe/public/js/frappe/utils/number_systems.js index 8224f11d37..ee04dd8271 100644 --- a/frappe/public/js/frappe/utils/number_systems.js +++ b/frappe/public/js/frappe/utils/number_systems.js @@ -31,4 +31,26 @@ export default { symbol: __("K", null, "Number system"), }, ], + nepalese: [ + { + divisor: 1.0e11, + symbol: __("Kh", null, "Number system"), // 10^11 is read as 1 Kharba + }, + { + divisor: 1.0e9, + symbol: __("Ar", null, "Number system"), // 10^9 is read as 1 Arba + }, + { + divisor: 1.0e7, + symbol: __("Cr", null, "Number system"), + }, + { + divisor: 1.0e5, + symbol: __("L", null, "Number system"), + }, + { + divisor: 1.0e3, + symbol: __("K", null, "Number system"), + }, + ], }; diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index e94452da4e..02305870cd 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1170,8 +1170,10 @@ Object.assign(frappe.utils, { }, get_number_system: function (country) { - if (["Nepal", "Bangladesh", "India", "Myanmar", "Pakistan"].includes(country)) { + if (["Bangladesh", "India", "Myanmar", "Pakistan"].includes(country)) { return number_systems.indian; + } else if (country == "Nepal") { + return number_systems.nepalese; } else { return number_systems.default; } From f87b52a93c14bf5a88180c29a032427ade7fcb1d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 6 Dec 2023 19:20:54 +0530 Subject: [PATCH 060/237] fix: set correct sentry tags --- frappe/public/js/sentry.bundle.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/js/sentry.bundle.js b/frappe/public/js/sentry.bundle.js index 0e9ba2fc69..edef9bc363 100644 --- a/frappe/public/js/sentry.bundle.js +++ b/frappe/public/js/sentry.bundle.js @@ -7,5 +7,6 @@ Sentry.init({ initialScope: { // don't use frappe.session.user, it's set much later and will fail because of async loading user: { id: frappe.boot.user.name ?? "Unidentified" }, + tags: { site: frappe.boot.sitename }, }, }); From d74e5d2e8a747a56460a6715ca4a5a3b0e19aa10 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Thu, 7 Dec 2023 09:31:53 +0530 Subject: [PATCH 061/237] chore: fix typo in comment --- frappe/model/mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index 7363bf4583..f6b710064d 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -16,7 +16,7 @@ def make_mapped_doc(method, source_name, selected_children=None, args=None): Called from `open_mapped_doc` from create_new.js""" for hook in reversed(frappe.get_hooks("override_whitelisted_methods", {}).get(method, [])): - # override using the first hook + # override using the last hook method = hook break From 61425cf8581a5e4d487c0c5eaa83719e9b91ebd1 Mon Sep 17 00:00:00 2001 From: Trusted Computer <75872475+trustedcomputer@users.noreply.github.com> Date: Wed, 6 Dec 2023 20:52:13 -0800 Subject: [PATCH 062/237] fix: remove redundant breadcrumbs Remove redundant frappe.breadcrumbs.add from notifications_settings --- .../doctype/notification_settings/notification_settings.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frappe/desk/doctype/notification_settings/notification_settings.js b/frappe/desk/doctype/notification_settings/notification_settings.js index ba72369273..35c94025a3 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.js +++ b/frappe/desk/doctype/notification_settings/notification_settings.js @@ -3,11 +3,6 @@ frappe.ui.form.on("Notification Settings", { onload: (frm) => { - frappe.breadcrumbs.add({ - label: __("Settings"), - route: "#modules/Settings", - type: "Custom", - }); frm.set_query("subscribed_documents", () => { return { filters: { From 4c9dea8d62233848d7f370db67053e8c88f41801 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 7 Dec 2023 12:46:27 +0530 Subject: [PATCH 063/237] fix: use encoded filename for arabic (latin) language while exporting --- frappe/utils/response.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/utils/response.py b/frappe/utils/response.py index 61372eee23..cce341f926 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -145,7 +145,9 @@ def as_pdf(): def as_binary(): response = Response() response.mimetype = "application/octet-stream" - response.headers.add("Content-Disposition", None, filename=frappe.response["filename"]) + filename = frappe.response["filename"] + filename = filename.encode("utf-8").decode("unicode-escape", "ignore") + response.headers.add("Content-Disposition", None, filename=filename) response.data = frappe.response["filecontent"] return response From f679f65aa5df29c9d0320d77178ba16966bef9b8 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 7 Dec 2023 13:42:24 +0530 Subject: [PATCH 064/237] test: unit test to check arabic filename export --- frappe/tests/test_api.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py index a5c76e2698..8d3065982b 100644 --- a/frappe/tests/test_api.py +++ b/frappe/tests/test_api.py @@ -419,6 +419,19 @@ class TestResponse(FrappeAPITestCase): self.assertIn("text/csv", response.headers["content-type"]) self.assertGreater(cint(response.headers["content-length"]), 0) + from frappe.desk.utils import provide_binary_file + from frappe.utils.response import build_response + + filename = "دفتر الأستاذ العام" + encoded_filename = filename.encode("utf-8").decode("unicode-escape", "ignore") + ".xlsx" + provide_binary_file(filename, "xlsx", "content") + + response = build_response("binary") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers["content-type"], "application/octet-stream") + self.assertGreater(cint(response.headers["content-length"]), 0) + self.assertEqual(response.headers["content-disposition"], f'filename="{encoded_filename}"') + def generate_admin_keys(): from frappe.core.doctype.user.user import generate_keys From b483800a06bf5c988881686f06b89ef122c2ee3b Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 7 Dec 2023 19:23:35 +0530 Subject: [PATCH 065/237] fix: pass filter to formatter to handle custom logic --- frappe/public/js/frappe/views/reports/query_report.js | 8 +++----- package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index a31d1c21a1..37c1cd6ea8 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1259,17 +1259,15 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { width: parseInt(column.width) || null, editable: false, compareValue: compareFn, - format: (value, row, column, data, for_filter = false) => { - if (for_filter && column?.fieldtype === "Link") { - return value || ""; - } + format: (value, row, column, data, filter) => { if (this.report_settings.formatter) { return this.report_settings.formatter( value, row, column, data, - format_cell + format_cell, + filter ); } return format_cell(value, row, column, data); diff --git a/package.json b/package.json index 8e53dcb29a..cfde368887 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "fast-deep-equal": "^2.0.1", "fast-glob": "^3.2.5", "frappe-charts": "2.0.0-rc22", - "frappe-datatable": "^1.17.2", + "frappe-datatable": "^1.17.5", "frappe-gantt": "^0.6.0", "highlight.js": "^10.4.1", "html5-qrcode": "^2.3.8", diff --git a/yarn.lock b/yarn.lock index 68fcad21cf..f6045227c5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1497,10 +1497,10 @@ frappe-charts@2.0.0-rc22: resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc22.tgz#9a5a747febdc381a1d4d7af96e89cf519dfba8c0" integrity sha512-N7f/8979wJCKjusOinaUYfMxB80YnfuVLrSkjpj4LtyqS0BGS6SuJxUnb7Jl4RWUFEIs7zEhideIKnyLeFZF4Q== -frappe-datatable@^1.17.2: - version "1.17.4" - resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.17.4.tgz#b003a8097f462fa5933f236590865be0cf370b44" - integrity sha512-uqtTzgdYITZM8hDvTEnmYnYRmPsoG+AjbTDls/NMxMBCkzRAE0VTVaPs5yag7HWoYJg/9LFW+sHwgZbVpjyBEA== +frappe-datatable@^1.17.5: + version "1.17.5" + resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.17.5.tgz#342814d7f9eb502f55639c1b09d44d9eca782f78" + integrity sha512-FJdpsj/xACuk553FXMMqvtTQZII9P9TEtAwOiaKN+AwDAjfCO4b5vhSTmdNp2Kgf26bnZU9QVfRvh2gDtpj3OA== dependencies: hyperlist "^1.0.0-beta" lodash "^4.17.5" From cfc781e5b67a2513d296a5aabc17fb341def3065 Mon Sep 17 00:00:00 2001 From: Md Hussain Nagaria <34810212+NagariaHussain@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:58:05 +0530 Subject: [PATCH 066/237] fix(Virtual DocType): don't call db get_value in link field validation (#23620) * fix(Virtual DocType): use get_doc to validate link field instead of db get_value * refactor: use is_virtual_doctype util & always return a dict --- frappe/client.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/client.py b/frappe/client.py index 91f531fe1e..c65753c766 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -10,6 +10,7 @@ import frappe.utils from frappe import _ from frappe.desk.reportview import validate_args from frappe.model.db_query import check_parent_permission +from frappe.model.utils import is_virtual_doctype from frappe.utils import get_safe_filters from frappe.utils.deprecations import deprecated @@ -431,6 +432,18 @@ def validate_link(doctype: str, docname: str, fields=None): ) values = frappe._dict() + + if is_virtual_doctype(doctype): + try: + frappe.get_doc(doctype, docname) + values.name = docname + except frappe.DoesNotExistError: + frappe.clear_last_message() + frappe.msgprint( + _("Document {0} {1} does not exist").format(frappe.bold(doctype), frappe.bold(docname)), + ) + return values + values.name = frappe.db.get_value(doctype, docname, cache=True) fields = frappe.parse_json(fields) From 78cf0cd142faca841adf668c98ca4a32f9b8ea7c Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 8 Dec 2023 10:53:58 +0530 Subject: [PATCH 067/237] fix: check if autoname is promt before setting __newname --- frappe/model/document.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 8eb45e8827..dd52262f64 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -464,8 +464,11 @@ class Document(BaseDocument): if self.flags.name_set and not force: return + meta = self.meta or frappe.get_meta(self.doctype) + autoname = meta.autoname or "" + # If autoname has set as Prompt (name) - if self.get("__newname"): + if self.get("__newname") and autoname == "Prompt": self.name = validate_name(self.doctype, self.get("__newname")) self.flags.name_set = True return From b04a6247ce566cc40e04ad352fa9c50b2e18c513 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 8 Dec 2023 11:04:30 +0530 Subject: [PATCH 068/237] fix: Color indicator style --- frappe/public/scss/common/color_picker.scss | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/public/scss/common/color_picker.scss b/frappe/public/scss/common/color_picker.scss index f80057f5b3..6e4b8e9d23 100644 --- a/frappe/public/scss/common/color_picker.scss +++ b/frappe/public/scss/common/color_picker.scss @@ -96,19 +96,19 @@ .frappe-control[data-fieldtype="Color"] { input { - padding-left: 38px; + padding-left: 32px; } .control-input { position: relative; } .selected-color { cursor: pointer; - width: 22px; - height: 22px; + width: 16px; + height: 16px; border-radius: 5px; background-color: red; position: absolute; - top: 5px; + top: 6px; left: 8px; content: " "; &.no-value { From f54f2419bbd6fa4c15f2cb47a6784f964fb93537 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 8 Dec 2023 13:12:05 +0530 Subject: [PATCH 069/237] fix: prioritize link_title if value != link_title --- frappe/public/js/frappe/form/formatters.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index aee19cd2f5..8e98c1042c 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -149,6 +149,10 @@ frappe.form.formatters = { var original_value = value; let link_title = frappe.utils.get_link_title(doctype, value); + if (link_title === value) { + link_title = null; + } + if (value && value.match && value.match(/^['"].*['"]$/)) { value.replace(/^.(.*).$/, "$1"); } From 95e161389b094176de1bfb7549577df46a9fad9b Mon Sep 17 00:00:00 2001 From: Ponnusamy <95607086+Ponnusamy1-V@users.noreply.github.com> Date: Fri, 8 Dec 2023 13:57:46 +0530 Subject: [PATCH 070/237] chore: typo (#23675) --- frappe/public/js/frappe/socketio_client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/socketio_client.js b/frappe/public/js/frappe/socketio_client.js index 4797756baa..ea5d64334d 100644 --- a/frappe/public/js/frappe/socketio_client.js +++ b/frappe/public/js/frappe/socketio_client.js @@ -214,5 +214,5 @@ class RealTimeClient { frappe.realtime = new RealTimeClient(); -// backward compatbility +// backward compatibility frappe.socketio = frappe.realtime; From 57699a54b176ca31f5ea7a9eb0841f09df819651 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 8 Dec 2023 15:01:13 +0530 Subject: [PATCH 071/237] fix: Show server script name in traceback (#23676) * fix: Show server script name in traceback * chore: typo Co-authored-by: Sagar Vora --------- Co-authored-by: Sagar Vora --- frappe/core/doctype/report/report.py | 2 +- .../doctype/server_script/server_script.py | 13 ++++++--- .../doctype/system_console/system_console.py | 2 +- frappe/query_builder/utils.py | 27 ++++++++++--------- frappe/recorder.py | 4 ++- frappe/utils/error.py | 7 ++--- frappe/utils/safe_exec.py | 16 +++++++++-- frappe/website/doctype/web_page/web_page.py | 2 +- 8 files changed, 47 insertions(+), 26 deletions(-) diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index f43bc65f6b..ad51b0415c 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -184,7 +184,7 @@ class Report(Document): def execute_script(self, filters): # server script loc = {"filters": frappe._dict(filters), "data": None, "result": None} - safe_exec(self.report_script, None, loc) + safe_exec(self.report_script, None, loc, script_filename=f"Report {self.name}") if loc["data"]: return loc["data"] else: diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index a58e50dddc..e19bdc681d 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -155,7 +155,12 @@ class ServerScript(Document): Args: doc (Document): Executes script with for a certain document's events """ - safe_exec(self.script, _locals={"doc": doc}, restrict_commit_rollback=True) + safe_exec( + self.script, + _locals={"doc": doc}, + restrict_commit_rollback=True, + script_filename=self.name, + ) def execute_scheduled_method(self): """Specific to Scheduled Jobs via Server Scripts @@ -166,7 +171,7 @@ class ServerScript(Document): if self.script_type != "Scheduler Event": raise frappe.DoesNotExistError - safe_exec(self.script) + safe_exec(self.script, script_filename=self.name) def get_permission_query_conditions(self, user: str) -> list[str]: """Specific to Permission Query Server Scripts @@ -178,7 +183,7 @@ class ServerScript(Document): list: Returns list of conditions defined by rules in self.script """ locals = {"user": user, "conditions": ""} - safe_exec(self.script, None, locals) + safe_exec(self.script, None, locals, script_filename=self.name) if locals["conditions"]: return locals["conditions"] @@ -278,7 +283,7 @@ def execute_api_server_script(script=None, *args, **kwargs): raise frappe.PermissionError # output can be stored in flags - _globals, _locals = safe_exec(script.script) + _globals, _locals = safe_exec(script.script, script_filename=script.name) return _globals.frappe.flags diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py index 4969f5a04a..f34f750f9c 100644 --- a/frappe/desk/doctype/system_console/system_console.py +++ b/frappe/desk/doctype/system_console/system_console.py @@ -28,7 +28,7 @@ class SystemConsole(Document): try: frappe.local.debug_log = [] if self.type == "Python": - safe_exec(self.console) + safe_exec(self.console, script_filename="System Console") self.output = "\n".join(frappe.debug_log) elif self.type == "SQL": self.output = frappe.as_json(read_sql(self.console, as_dict=1)) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index c7000a0409..f4ab85ac6b 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -107,6 +107,8 @@ def patch_query_execute(): def prepare_query(query): import inspect + from frappe.utils.safe_exec import SERVER_SCRIPT_FILE_PREFIX + param_collector = NamedParameterWrapper() query = query.get_sql(param_wrapper=param_collector) if frappe.flags.in_safe_exec: @@ -114,21 +116,20 @@ def patch_query_execute(): if not check_safe_sql_query(query, throw=False): callstack = inspect.stack() - if len(callstack) >= 3 and ".py" in callstack[2].filename: - # ignore any query builder methods called from python files - # assumption is that those functions are whitelisted already. - # since query objects are patched everywhere any query.run() - # will have callstack like this: - # frame0: this function prepare_query() - # frame1: execute_query() - # frame2: frame that called `query.run()` - # - # if frame2 is server script is set as the filename - # it shouldn't be allowed. - pass - else: + # This check is required because QB can execute from anywhere and we can not + # reliably provide a safe version for it in server scripts. + + # since query objects are patched everywhere any query.run() + # will have callstack like this: + # frame0: this function prepare_query() + # frame1: execute_query() + # frame2: frame that called `query.run()` + # + # if frame2 is server script is set as the filename it shouldn't be allowed. + if len(callstack) >= 3 and SERVER_SCRIPT_FILE_PREFIX in callstack[2].filename: raise frappe.PermissionError("Only SELECT SQL allowed in scripting") + return query, param_collector.get_parameters() builder_class = frappe.qb._BuilderClasss diff --git a/frappe/recorder.py b/frappe/recorder.py index 68356c732e..9e67ca9ff0 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -41,11 +41,13 @@ def sql(*args, **kwargs): def get_current_stack_frames(): + from frappe.utils.safe_exec import SERVER_SCRIPT_FILE_PREFIX + try: current = inspect.currentframe() frames = inspect.getouterframes(current, context=10) for frame, filename, lineno, function, context, index in list(reversed(frames))[:-2]: - if "/apps/" in filename or "" in filename: + if "/apps/" in filename or SERVER_SCRIPT_FILE_PREFIX in filename: yield { "filename": TRACEBACK_PATH_PATTERN.sub("", filename), "lineno": lineno, diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 50432a01d5..6cee88598e 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -146,19 +146,20 @@ def guess_exception_source(exception: str) -> str | None: - For unhandled exception last python file from apps folder is responsible. - For frappe.throws the exception source is possibly present after skipping frappe.throw frames - - For server script the file name is `` + - For server script the file name contains SERVER_SCRIPT_FILE_PREFIX """ + from frappe.utils.safe_exec import SERVER_SCRIPT_FILE_PREFIX + with suppress(Exception): installed_apps = frappe.get_installed_apps() app_priority = {app: installed_apps.index(app) for app in installed_apps} APP_NAME_REGEX = re.compile(r".*File.*apps/(?P\w+)/\1/") - SERVER_SCRIPT_FRAME = re.compile(r".*") apps = Counter() for line in reversed(exception.splitlines()): - if SERVER_SCRIPT_FRAME.match(line): + if SERVER_SCRIPT_FILE_PREFIX in line: return "Server Script" if matches := APP_NAME_REGEX.match(line): diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 6bdf34bc77..a9f457a0a6 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -37,6 +37,7 @@ class ServerScriptNotEnabled(frappe.PermissionError): ARGUMENT_NOT_SET = object() SAFE_EXEC_CONFIG_KEY = "server_script_enabled" +SERVER_SCRIPT_FILE_PREFIX = "" class NamespaceDict(frappe._dict): @@ -76,7 +77,14 @@ def is_safe_exec_enabled() -> bool: return bool(frappe.get_common_site_config().get(SAFE_EXEC_CONFIG_KEY)) -def safe_exec(script, _globals=None, _locals=None, restrict_commit_rollback=False): +def safe_exec( + script: str, + _globals: dict | None = None, + _locals: dict | None = None, + *, + restrict_commit_rollback: bool = False, + script_filename: str | None = None, +): if not is_safe_exec_enabled(): msg = _("Server Scripts are disabled. Please enable server scripts from bench configuration.") @@ -95,10 +103,14 @@ def safe_exec(script, _globals=None, _locals=None, restrict_commit_rollback=Fals exec_globals.frappe.db.pop("rollback", None) exec_globals.frappe.db.pop("add_index", None) + filename = SERVER_SCRIPT_FILE_PREFIX + if script_filename: + filename += f": {frappe.scrub(script_filename)}" + with safe_exec_flags(), patched_qb(): # execute script compiled by RestrictedPython exec( - compile_restricted(script, filename="", policy=FrappeTransformer), + compile_restricted(script, filename=filename, policy=FrappeTransformer), exec_globals, _locals, ) diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index 5bcc237e60..12f86f44c2 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -78,7 +78,7 @@ class WebPage(WebsiteGenerator): if self.context_script: _locals = dict(context=frappe._dict()) - safe_exec(self.context_script, None, _locals) + safe_exec(self.context_script, None, _locals, script_filename=f"web page {self.name}") context.update(_locals["context"]) self.render_dynamic(context) From adbd88d3946673717e1996a260d2e31d0eb254d0 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 8 Dec 2023 14:57:59 +0530 Subject: [PATCH 072/237] test: fixed failing test --- frappe/tests/test_db_query.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 5227d7ab4e..c5fdb8e10c 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -104,6 +104,7 @@ class TestDBQuery(FrappeTestCase): "doctype": "DocType", "name": "Parent DocType 1", "module": "Custom", + "autoname": "Prompt", "custom": 1, "fields": [ {"label": "Title", "fieldname": "title", "fieldtype": "Data"}, @@ -122,6 +123,7 @@ class TestDBQuery(FrappeTestCase): { "doctype": "DocType", "name": "Parent DocType 2", + "autoname": "Prompt", "module": "Custom", "custom": 1, "fields": [ From 52cea3e01cafe14fe42c1e8dcd6396521fe4b0ae Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 8 Dec 2023 15:49:33 +0530 Subject: [PATCH 073/237] chore: better code --- frappe/model/document.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index dd52262f64..da0f2aca87 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -464,11 +464,10 @@ class Document(BaseDocument): if self.flags.name_set and not force: return - meta = self.meta or frappe.get_meta(self.doctype) - autoname = meta.autoname or "" + autoname = self.meta.autoname or "" # If autoname has set as Prompt (name) - if self.get("__newname") and autoname == "Prompt": + if self.get("__newname") and autoname.lower() == "prompt": self.name = validate_name(self.doctype, self.get("__newname")) self.flags.name_set = True return From faef64bab420b945b2b333ae3c7a2b721689bde3 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Fri, 8 Dec 2023 16:13:15 +0530 Subject: [PATCH 074/237] fix(sentry): set the user's name correctly https://docs.sentry.io/platforms/python/enriching-events/identify-user/ Signed-off-by: Akhil Narang --- frappe/utils/sentry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/sentry.py b/frappe/utils/sentry.py index 4b01b2996f..9a23508d4e 100644 --- a/frappe/utils/sentry.py +++ b/frappe/utils/sentry.py @@ -41,7 +41,7 @@ def capture_exception( evt_processor = _make_wsgi_event_processor(frappe.request.environ, False) scope.add_event_processor(evt_processor) scope.set_tag("site", frappe.local.site) - scope.set_tag("user", getattr(frappe.session, "user", "Unidentified")) + scope.set_user({"name": getattr(frappe.session, "user", "Unidentified")}) # Extract `X-Frappe-Request-ID` to store as a separate field if its present if trace_id := frappe.monitor.get_trace_id(): From 01d6679c8c4fbe781ad94340180318a311dc0d0d Mon Sep 17 00:00:00 2001 From: sibi kumar k <95605794+sibikumarkuppusamy@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:47:23 +0530 Subject: [PATCH 075/237] fix: Add filter in Workspace Card Links Block (#23578) --- frappe/public/js/frappe/widgets/widget_dialog.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index 14ce39355a..2065b59428 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -259,6 +259,15 @@ class CardDialog extends WidgetDialog { get_options: (df) => { return df.doc.link_type; }, + get_query: function (df) { + if (df.link_type == "DocType") { + return { + filters: { + istable: 0, + }, + }; + } + }, }, { fieldname: "column_break_7", From 84fcd857e4ea995d8a6426a2d22305c6d3c50711 Mon Sep 17 00:00:00 2001 From: Fisher Yu <12823863+szufisher@users.noreply.github.com> Date: Fri, 8 Dec 2023 19:26:40 +0800 Subject: [PATCH 076/237] feat(minor): child table field as standard filter in list view (#23625) --- frappe/public/js/frappe/list/base_list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index e7c0cffda8..a9659d079f 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -831,7 +831,7 @@ class FilterArea { value = "%" + value + "%"; } filters.push([ - this.list_view.doctype, + field.df.doctype || this.list_view.doctype, field.df.fieldname, field.df.condition || "=", value, From bfc2c07630ec271525f82e896bf0ec7ffd6cc745 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:57:18 +0530 Subject: [PATCH 077/237] fix: undefined variable name fixed in dark.scss (backport #23646) (#23693) Co-authored-by: MasterCat --- frappe/public/scss/desk/dark.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/scss/desk/dark.scss b/frappe/public/scss/desk/dark.scss index d168f4247a..027b355d8e 100644 --- a/frappe/public/scss/desk/dark.scss +++ b/frappe/public/scss/desk/dark.scss @@ -19,7 +19,7 @@ $check-icon-dark: url("data:image/svg+xml, Date: Fri, 8 Dec 2023 13:40:51 +0100 Subject: [PATCH 078/237] fix: revert wrong cleanup attempt --- frappe/public/js/frappe/list/list_view.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 0ee456a049..929c7b9812 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -408,7 +408,11 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { type: "Tag", }); - if (!this.settings.hide_name_column) { + if ( + !this.settings.hide_name_column && + this.meta.title_field && + this.meta.title_field !== "name" + ) { this.columns.push({ type: "Field", df: { From 7452e71d02b991b0b1a578178814db5a2bd22e8b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 8 Dec 2023 20:55:25 +0530 Subject: [PATCH 079/237] perf: Use get_cached_doc instead of get_doc - Makes document pages with no-cache load a bit faster --- frappe/website/page_renderers/document_page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index 6cd59eb6a1..83d55f7a9a 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -44,7 +44,7 @@ class DocumentPage(BaseTemplatePage): @cache_html def get_html(self): - self.doc = frappe.get_doc(self.doctype, self.docname) + self.doc = frappe.get_cached_doc(self.doctype, self.docname) self.init_context() self.update_context() self.post_process_context() From c0de1aa0df4bd8ab76f51ec92436f20e56faa685 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Fri, 8 Dec 2023 21:07:17 +0530 Subject: [PATCH 080/237] refactor(sentry): don't require passing in an exception (#23689) `sys.exc_info()` works out for our use case Signed-off-by: Akhil Narang --- frappe/utils/sentry.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/frappe/utils/sentry.py b/frappe/utils/sentry.py index 9a23508d4e..c7843d8c10 100644 --- a/frappe/utils/sentry.py +++ b/frappe/utils/sentry.py @@ -16,13 +16,10 @@ def before_send(event, hint): return event -def capture_exception( - exception: ValueError | BaseException | None = None, message: str | None = None -) -> None: +def capture_exception(message: str | None = None) -> None: """ Function to upload exception data to entry - :param exception: Exception object - if missing, try to get with sys.exc_info() :param message: A message to be sent if we can't find an exception """ # Don't report anything if the user hasn't opted-in to telemetry @@ -48,17 +45,16 @@ def capture_exception( scope.set_tag("frappe_trace_id", trace_id) if client := hub.client: - if exception is None and ((exception := sys.exc_info()[1]) is None): - if message: - sentry_capture_message(message, level="error") - return - - event, hint = event_from_exception( - exception, - client_options=client.options, - mechanism={"type": "wsgi", "handled": False}, - ) - hub.capture_event(event, hint=hint) + exc_info = sys.exc_info() + if any(exc_info): + event, hint = event_from_exception( + exc_info, + client_options=client.options, + mechanism={"type": "wsgi", "handled": False}, + ) + hub.capture_event(event, hint=hint) + elif message: + sentry_capture_message(message, level="error") except Exception: frappe.logger().error("Failed to capture exception", exc_info=True) From bc2b41d2f5dc01d8d47b3efdf122bbd367a94e25 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 8 Dec 2023 21:15:34 +0530 Subject: [PATCH 081/237] perf: Simplify dynamic route evaluation to speed up route resolution validate complete route only if starting of the path matches with the webform route This avoids the expensive matching from huge route_map (in sites with lots of web forms) --- frappe/website/doctype/web_form/web_form.py | 2 +- frappe/website/router.py | 28 +++++++++++---------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 4e7eac423d..2b2f3743f5 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -279,7 +279,7 @@ def get_context(context): if field.fieldtype == "Select" and field.options: messages.extend(field.options.split("\n")) - messages.extend(col.label for col in self.list_columns) + messages.extend(col.get("label") if col else "" for col in self.list_columns) context.translated_messages = frappe.as_json( {message: _(message) for message in messages if message} diff --git a/frappe/website/router.py b/frappe/website/router.py index 3085212aa1..2947b89f91 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -34,27 +34,29 @@ def get_page_info_from_web_form(path): """Query published web forms and evaluate if the route matches""" from frappe.website.doctype.web_form.web_form import get_published_web_forms - rules, page_info = [], {} for d in get_published_web_forms(): + if not (path.startswith(f"{d.route}") or path.startswith(f"/{d.route}")): + continue + + rules = [] rules.append(Rule(f"/{d.route}", endpoint=d.name)) rules.append(Rule(f"/{d.route}/list", endpoint=d.name)) rules.append(Rule(f"/{d.route}/new", endpoint=d.name)) rules.append(Rule(f"/{d.route}/", endpoint=d.name)) rules.append(Rule(f"/{d.route}//edit", endpoint=d.name)) d.doctype = "Web Form" - page_info[d.name] = d + end_point = evaluate_dynamic_routes(rules, path) - end_point = evaluate_dynamic_routes(rules, path) - if end_point: - if path.endswith("/list"): - frappe.form_dict.is_list = True - elif path.endswith("/new"): - frappe.form_dict.is_new = True - elif path.endswith("/edit"): - frappe.form_dict.is_edit = True - else: - frappe.form_dict.is_read = True - return page_info[end_point] + if end_point: + if path.endswith("/list"): + frappe.form_dict.is_list = True + elif path.endswith("/new"): + frappe.form_dict.is_new = True + elif path.endswith("/edit"): + frappe.form_dict.is_edit = True + else: + frappe.form_dict.is_read = True + return d def evaluate_dynamic_routes(rules, path): From 4b367245ebc4f40656759c8248ada52eeb3d1aa5 Mon Sep 17 00:00:00 2001 From: Corentin Flr <10946971+cogk@users.noreply.github.com> Date: Mon, 11 Dec 2023 06:10:27 +0100 Subject: [PATCH 082/237] Merge pull request from GHSA-v3vh-7qx4-f582 --- frappe/utils/safe_exec.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index a9f457a0a6..b2cea7f7cf 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -149,9 +149,16 @@ def _validate_safe_eval_syntax(code): @contextmanager def safe_exec_flags(): - frappe.flags.in_safe_exec = True - yield - frappe.flags.in_safe_exec = False + if not frappe.flags.in_safe_exec: + frappe.flags.in_safe_exec = 0 + + frappe.flags.in_safe_exec += 1 + + try: + yield + finally: + # Always ensure that the flag is decremented + frappe.flags.in_safe_exec -= 1 def get_safe_globals(): From 8cc4fc349acb420a4193360bcc3a5e9f8164db24 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 11 Dec 2023 10:53:04 +0530 Subject: [PATCH 083/237] chore: improve condition to avoid re-initializing flag --- frappe/utils/safe_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index b2cea7f7cf..b4581d8387 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -149,7 +149,7 @@ def _validate_safe_eval_syntax(code): @contextmanager def safe_exec_flags(): - if not frappe.flags.in_safe_exec: + if frappe.flags.in_safe_exec is None: frappe.flags.in_safe_exec = 0 frappe.flags.in_safe_exec += 1 From 81b75374b05c34c4b87202ae09953e68bff040d9 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 11 Dec 2023 11:09:49 +0530 Subject: [PATCH 084/237] fix: escape username on avatar control (#23713) [skip ci] --- frappe/templates/includes/avatar_macro.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/templates/includes/avatar_macro.html b/frappe/templates/includes/avatar_macro.html index 7313711c6b..5db1654ef5 100644 --- a/frappe/templates/includes/avatar_macro.html +++ b/frappe/templates/includes/avatar_macro.html @@ -1,15 +1,15 @@ {% macro avatar(user_id=None, css_style=None, size="avatar-small", full_name=None, image=None) %} {% set user_info = frappe.utils.get_user_info_for_avatar(user_id) %} - + {% if image or user_info.image %} + title="{{ full_name|e or user_info.name|e }}"> {% else %} + title="{{ full_name|e or user_info.name|e }}"> {{ frappe.utils.get_abbr(full_name or user_info.name).upper() }} {% endif %} From 7027e8b4dcdf96f642f6c92acd10c21ba8e56979 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 11 Dec 2023 11:20:02 +0530 Subject: [PATCH 085/237] fix: ecsape search string (#23717) [skip ci] --- frappe/templates/includes/list/list.html | 2 +- frappe/templates/includes/list/list.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/templates/includes/list/list.html b/frappe/templates/includes/list/list.html index 9575344a70..188df55bd5 100644 --- a/frappe/templates/includes/list/list.html +++ b/frappe/templates/includes/list/list.html @@ -8,7 +8,7 @@ {% else %}
+ data-txt="{{ txt|e or '[notxt]' }}"> diff --git a/frappe/templates/includes/list/list.js b/frappe/templates/includes/list/list.js index 6bd3a155f2..27abe7c393 100644 --- a/frappe/templates/includes/list/list.js +++ b/frappe/templates/includes/list/list.js @@ -6,7 +6,7 @@ frappe.ready(function() { var btn = $(this); var data = $.extend(frappe.utils.get_query_params(), { doctype: "{{ doctype }}", - txt: "{{ txt or '' }}", + txt: "{{ txt|e or '' }}", limit_start: next_start, pathname: location.pathname, }); From e4a9431c9c2f31e322ca0667a37fb607f39269a8 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 11 Dec 2023 12:00:01 +0530 Subject: [PATCH 086/237] fix: change webhook header types to small text (#23722) closes https://github.com/frappe/frappe/issues/18650 --- .../integrations/doctype/webhook_header/webhook_header.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/integrations/doctype/webhook_header/webhook_header.json b/frappe/integrations/doctype/webhook_header/webhook_header.json index 4aea5d02ed..6a7e8f9999 100644 --- a/frappe/integrations/doctype/webhook_header/webhook_header.json +++ b/frappe/integrations/doctype/webhook_header/webhook_header.json @@ -11,20 +11,20 @@ "fields": [ { "fieldname": "key", - "fieldtype": "Data", + "fieldtype": "Small Text", "in_list_view": 1, "label": "Key" }, { "fieldname": "value", - "fieldtype": "Data", + "fieldtype": "Small Text", "in_list_view": 1, "label": "Value" } ], "istable": 1, "links": [], - "modified": "2022-08-03 12:20:51.949422", + "modified": "2023-12-11 12:20:51.949422", "modified_by": "Administrator", "module": "Integrations", "name": "Webhook Header", From f0aaeadd498a9d730f29a1d2753bf63266c8a6c0 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 11 Dec 2023 12:48:47 +0530 Subject: [PATCH 087/237] fix: Correct fallback for social login key --- .../integrations/doctype/social_login_key/social_login_key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/integrations/doctype/social_login_key/social_login_key.py b/frappe/integrations/doctype/social_login_key/social_login_key.py index 3445bb92e3..54f2c3ae1b 100644 --- a/frappe/integrations/doctype/social_login_key/social_login_key.py +++ b/frappe/integrations/doctype/social_login_key/social_login_key.py @@ -223,5 +223,5 @@ def provider_allows_signup(provider: str) -> bool: sign_up_config = frappe.db.get_value("Social Login Key", provider, "sign_ups") if not sign_up_config: # fallback to global settings - return is_signup_disabled() + return not is_signup_disabled() return sign_up_config == "Allow" From 8129fe7268efecfb238aab62dba51755087d6257 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 11 Dec 2023 17:15:37 +0530 Subject: [PATCH 088/237] fix: Avoid duplicate contact names (#23731) --- frappe/contacts/doctype/contact/contact.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index c1bd2f55ec..5eaf28def0 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -50,14 +50,14 @@ class Contact(Document): def autoname(self): self.name = self._get_full_name() - if frappe.db.exists("Contact", self.name): - self.name = append_number_if_name_exists("Contact", self.name) - # concat party name if reqd for link in self.links: self.name = self.name + "-" + link.link_name.strip() break + if frappe.db.exists("Contact", self.name): + self.name = append_number_if_name_exists("Contact", self.name) + def validate(self): self.full_name = self._get_full_name() self.set_primary_email() From 2f7b9f8a5a56a6a2c0fed7bab4bb54247cd28b86 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:07:30 +0100 Subject: [PATCH 089/237] fix: get users for leaderboard --- frappe/desk/leaderboard.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/desk/leaderboard.py b/frappe/desk/leaderboard.py index ff41019aa1..91aa2084cc 100644 --- a/frappe/desk/leaderboard.py +++ b/frappe/desk/leaderboard.py @@ -15,18 +15,17 @@ def get_leaderboards(): @frappe.whitelist() def get_energy_point_leaderboard(date_range, company=None, field=None, limit=None): - all_users = frappe.get_all( + users = frappe.get_list( "User", filters={ "name": ["not in", ["Administrator", "Guest"]], "enabled": 1, "user_type": ["!=", "Website User"], }, - order_by="name ASC", + pluck="name", ) - all_users_list = list(map(lambda x: x["name"], all_users)) - filters = [["type", "!=", "Review"], ["user", "in", all_users_list]] + filters = [["type", "!=", "Review"], ["user", "in", users]] if date_range: date_range = frappe.parse_json(date_range) filters.append(["creation", "between", [date_range[0], date_range[1]]]) @@ -39,7 +38,7 @@ def get_energy_point_leaderboard(date_range, company=None, field=None, limit=Non ) energy_point_users_list = list(map(lambda x: x["name"], energy_point_users)) - for user in all_users_list: + for user in users: if user not in energy_point_users_list: energy_point_users.append({"name": user, "value": 0}) From c648e4fc9a0ce6ac2e2a3c4bf2a4b7de1de92835 Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 11 Dec 2023 18:24:28 +0100 Subject: [PATCH 090/237] style: Linter --- frappe/tests/test_dashboard_connections.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/tests/test_dashboard_connections.py b/frappe/tests/test_dashboard_connections.py index 9182bd4f87..f0cdc675c6 100644 --- a/frappe/tests/test_dashboard_connections.py +++ b/frappe/tests/test_dashboard_connections.py @@ -122,7 +122,6 @@ class TestDashboardConnections(FrappeTestCase): expected_open_count, ) - def test_external_doctype_link_with_dashboard_override(self): # add a custom links todo = TestCustomizeForm().get_customize_form("ToDo") @@ -140,10 +139,12 @@ class TestDashboardConnections(FrappeTestCase): # Change standard fieldname, see if all custom links still work with patch_hooks( - {"override_doctype_dashboards": { - "ToDo": ["frappe.tests.test_dashboard_connections.get_dashboard_for_todo"] + { + "override_doctype_dashboards": { + "ToDo": ["frappe.tests.test_dashboard_connections.get_dashboard_for_todo"] + } } - }): + ): connections = get_open_count("ToDo", todo_doc.name)["count"] self.assertEqual(len(connections["external_links_found"]), 2) From f62f4472e651b5f2947321568fc949758b920420 Mon Sep 17 00:00:00 2001 From: Smit Vora Date: Mon, 11 Dec 2023 23:24:00 +0530 Subject: [PATCH 091/237] fix: ignore if controller doesn't have `get_list` attr (#23736) --- frappe/model/db_query.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index ee489da65f..7baa5848ed 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -179,6 +179,9 @@ class DatabaseQuery: from frappe.model.base_document import get_controller controller = get_controller(self.doctype) + if not hasattr(controller, "get_list"): + return [] + self.parse_args() kwargs = { "as_list": as_list, From fff96353284d38872a2bf6ee84dab233f531396d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 14:10:44 +0530 Subject: [PATCH 092/237] build(deps): bump actions/setup-python from 4 to 5 (#23743) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/linters.yml | 6 +++--- .github/workflows/on_release.yml | 2 +- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/server-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 5f2c5cfa29..8b35471d0c 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -38,7 +38,7 @@ jobs: steps: - name: 'Setup Environment' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' - uses: actions/checkout@v4 @@ -57,7 +57,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' cache: pip @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index e737c536bf..8979417c10 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -24,7 +24,7 @@ jobs: with: node-version: 18 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - name: Set up bench and build assets diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index d249c4a3b3..f48df1367a 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -62,7 +62,7 @@ jobs: fi - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index b3f2c5b0ca..5a7025acff 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 18 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' - name: Set up bench and build assets diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 351958de07..572253bdfa 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -83,7 +83,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index cbc0f74470..0bfde0fc25 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -65,7 +65,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' From 7edb80bf5cb96700aaa14a2f6a7abe09afde4bbc Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Dec 2023 14:49:37 +0530 Subject: [PATCH 093/237] fix: correct error message (#23746) [skip ci] --- frappe/custom/doctype/custom_field/custom_field.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index c77d2f4bb2..e84e0dd712 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -357,7 +357,7 @@ def rename_fieldname(custom_field: str, fieldname: str): if field.is_system_generated: frappe.throw(_("System Generated Fields can not be renamed")) if frappe.db.has_column(parent_doctype, fieldname): - frappe.throw(_("Can not rename as fieldname {0} is already present on DocType.")) + frappe.throw(_("Can not rename as column {0} is already present on DocType.").format(fieldname)) if old_fieldname == new_fieldname: frappe.msgprint(_("Old and new fieldnames are same."), alert=True) return From 21261e63e6d66a43fde595d6edeb5fa74efe832e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Dec 2023 21:43:19 +0530 Subject: [PATCH 094/237] fix: Workflow doc states not existing on new forms (#23756) [skip ci] --- frappe/workflow/doctype/workflow/workflow.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/workflow/doctype/workflow/workflow.js b/frappe/workflow/doctype/workflow/workflow.js index afca3d09b1..4bcffdde7a 100644 --- a/frappe/workflow/doctype/workflow/workflow.js +++ b/frappe/workflow/doctype/workflow/workflow.js @@ -49,13 +49,13 @@ frappe.ui.form.on("Workflow", { frm.events.update_field_options(frm); frm.ignore_warning = frm.is_new() ? true : false; frm.state_status_mapping = {}; - frm.doc.states.forEach((row) => { - frm.state_status_mapping[row.state] = row.doc_status; - }); if (frm.is_new()) { return; } + frm.doc.states.forEach((row) => { + frm.state_status_mapping[row.state] = row.doc_status; + }); frm.states = null; frm.trigger("make_state_table"); From 3a481c89e52b95588d6f2d3a301d5c65248c09d0 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:05:05 +0100 Subject: [PATCH 095/237] feat: add jinja syntax highlighting --- frappe/public/js/frappe/form/controls/code.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js index 42832ee6a1..db040d7c58 100644 --- a/frappe/public/js/frappe/form/controls/code.js +++ b/frappe/public/js/frappe/form/controls/code.js @@ -160,6 +160,7 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex JSON: "ace/mode/json", Golang: "ace/mode/golang", Go: "ace/mode/golang", + Jinja: "ace/mode/django", }; const language = this.df.options; From a586b5a1c665160fe191d7802e59af34c4de9941 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:05:58 +0100 Subject: [PATCH 096/237] feat(Email Template): switch from pure HTML to Jinja highlighting --- frappe/email/doctype/email_template/email_template.json | 4 ++-- frappe/email/doctype/email_template/email_template.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/email/doctype/email_template/email_template.json b/frappe/email/doctype/email_template/email_template.json index 0362a820e5..6bf08cd534 100644 --- a/frappe/email/doctype/email_template/email_template.json +++ b/frappe/email/doctype/email_template/email_template.json @@ -52,12 +52,12 @@ "fieldname": "response_html", "fieldtype": "Code", "label": "Response ", - "options": "HTML" + "options": "Jinja" } ], "icon": "fa fa-comment", "links": [], - "modified": "2023-08-28 22:29:04.457992", + "modified": "2023-12-12 20:01:07.080625", "modified_by": "Administrator", "module": "Email", "name": "Email Template", diff --git a/frappe/email/doctype/email_template/email_template.py b/frappe/email/doctype/email_template/email_template.py index 214879dac4..dead4a1c87 100644 --- a/frappe/email/doctype/email_template/email_template.py +++ b/frappe/email/doctype/email_template/email_template.py @@ -22,6 +22,7 @@ class EmailTemplate(Document): subject: DF.Data use_html: DF.Check # end: auto-generated types + @property def response_(self): return self.response_html if self.use_html else self.response From a83b5b324bf88221ed88af99c915d8581255796a Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:06:25 +0100 Subject: [PATCH 097/237] feat(Print Format): switch from pure HTML to Jinja highlighting --- frappe/printing/doctype/print_format/print_format.json | 7 ++++--- frappe/printing/doctype/print_format/print_format.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/printing/doctype/print_format/print_format.json b/frappe/printing/doctype/print_format/print_format.json index 8058700dbf..a58694f46d 100644 --- a/frappe/printing/doctype/print_format/print_format.json +++ b/frappe/printing/doctype/print_format/print_format.json @@ -112,14 +112,15 @@ "label": "HTML", "oldfieldname": "html", "oldfieldtype": "Text Editor", - "options": "HTML" + "options": "Jinja" }, { "depends_on": "raw_printing", "description": "Any string-based printer languages can be used. Writing raw commands requires knowledge of the printer's native language provided by the printer manufacturer. Please refer to the developer manual provided by the printer manufacturer on how to write their native commands. These commands are rendered on the server side using the Jinja Templating Language.", "fieldname": "raw_commands", "fieldtype": "Code", - "label": "Raw Commands" + "label": "Raw Commands", + "options": "Jinja" }, { "depends_on": "eval:!doc.custom_format", @@ -259,7 +260,7 @@ "icon": "fa fa-print", "idx": 1, "links": [], - "modified": "2023-08-28 20:25:09.660073", + "modified": "2023-12-12 19:59:37.133301", "modified_by": "Administrator", "module": "Printing", "name": "Print Format", diff --git a/frappe/printing/doctype/print_format/print_format.py b/frappe/printing/doctype/print_format/print_format.py index 426a2cd1a7..4110f849ec 100644 --- a/frappe/printing/doctype/print_format/print_format.py +++ b/frappe/printing/doctype/print_format/print_format.py @@ -48,6 +48,7 @@ class PrintFormat(Document): show_section_headings: DF.Check standard: DF.Literal["No", "Yes"] # end: auto-generated types + def onload(self): templates = frappe.get_all( "Print Format Field Template", From 468f3c6738cb8e2a55f1ab668ab4ff453a1519c3 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:06:45 +0100 Subject: [PATCH 098/237] feat(Web Template): switch from pure HTML to Jinja highlighting --- frappe/website/doctype/web_template/web_template.json | 5 +++-- frappe/website/doctype/web_template/web_template.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/website/doctype/web_template/web_template.json b/frappe/website/doctype/web_template/web_template.json index d2b909f950..719ca29e16 100644 --- a/frappe/website/doctype/web_template/web_template.json +++ b/frappe/website/doctype/web_template/web_template.json @@ -19,7 +19,7 @@ "fieldname": "template", "fieldtype": "Code", "label": "Template", - "options": "HTML" + "options": "Jinja" }, { "fieldname": "fields", @@ -54,7 +54,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-09-04 12:38:27.656042", + "modified": "2023-12-12 20:01:21.524022", "modified_by": "Administrator", "module": "Website", "name": "Web Template", @@ -76,5 +76,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/website/doctype/web_template/web_template.py b/frappe/website/doctype/web_template/web_template.py index 3a43c6037e..11a5e6b929 100644 --- a/frappe/website/doctype/web_template/web_template.py +++ b/frappe/website/doctype/web_template/web_template.py @@ -27,6 +27,7 @@ class WebTemplate(Document): template: DF.Code | None type: DF.Literal["Component", "Section", "Navbar", "Footer"] # end: auto-generated types + def validate(self): if self.standard and not frappe.conf.developer_mode and not frappe.flags.in_patch: frappe.throw(_("Enable developer mode to create a standard Web Template")) From 4f809630c5030e71df27f007f871528ac8f05431 Mon Sep 17 00:00:00 2001 From: Corentin Flr <10946971+cogk@users.noreply.github.com> Date: Wed, 13 Dec 2023 05:22:14 +0100 Subject: [PATCH 099/237] fix: Set sort_order to DESC if sort_field is modified (#23697) Global search and replace operation, including single doctypes where it does not really matter. --- frappe/automation/doctype/milestone/milestone.json | 4 ++-- .../doctype/milestone_tracker/milestone_tracker.json | 4 ++-- frappe/contacts/doctype/contact/contact.json | 4 ++-- frappe/core/doctype/error_log/error_log.json | 4 ++-- frappe/core/doctype/file/file.json | 4 ++-- frappe/core/doctype/module_def/module_def.json | 4 ++-- frappe/core/doctype/page/page.json | 4 ++-- frappe/core/doctype/role/role.json | 4 ++-- frappe/core/doctype/system_settings/system_settings.json | 4 ++-- frappe/core/doctype/version/version.json | 4 ++-- frappe/custom/doctype/client_script/client_script.json | 4 ++-- frappe/custom/doctype/custom_field/custom_field.json | 4 ++-- .../doctype/customize_form_field/customize_form_field.json | 4 ++-- frappe/desk/doctype/note/note.json | 4 ++-- .../integrations/doctype/google_contacts/google_contacts.json | 4 ++-- frappe/integrations/doctype/google_drive/google_drive.json | 4 ++-- frappe/printing/doctype/letter_head/letter_head.json | 4 ++-- frappe/social/doctype/review_level/review_level.json | 4 ++-- frappe/website/doctype/web_page/web_page.json | 4 ++-- frappe/website/doctype/website_settings/website_settings.json | 4 ++-- frappe/workflow/doctype/workflow/workflow.json | 4 ++-- frappe/workflow/doctype/workflow_state/workflow_state.json | 4 ++-- 22 files changed, 44 insertions(+), 44 deletions(-) diff --git a/frappe/automation/doctype/milestone/milestone.json b/frappe/automation/doctype/milestone/milestone.json index aa2dd35891..db3d61b2fd 100644 --- a/frappe/automation/doctype/milestone/milestone.json +++ b/frappe/automation/doctype/milestone/milestone.json @@ -53,7 +53,7 @@ ], "in_create": 1, "links": [], - "modified": "2022-08-03 12:20:55.076769", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Automation", "name": "Milestone", @@ -74,7 +74,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "title_field": "reference_type", "track_changes": 1 diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.json b/frappe/automation/doctype/milestone_tracker/milestone_tracker.json index 8d4ed94dcd..f0dc452c8d 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.json +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.json @@ -35,7 +35,7 @@ } ], "links": [], - "modified": "2022-08-03 12:20:54.955953", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Automation", "name": "Milestone Tracker", @@ -55,7 +55,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/contacts/doctype/contact/contact.json b/frappe/contacts/doctype/contact/contact.json index 679d8b4c8f..83d1002acb 100644 --- a/frappe/contacts/doctype/contact/contact.json +++ b/frappe/contacts/doctype/contact/contact.json @@ -257,7 +257,7 @@ "image_field": "image", "index_web_pages_for_search": 1, "links": [], - "modified": "2023-10-02 12:00:27.299156", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Contacts", "name": "Contact", @@ -392,7 +392,7 @@ ], "show_title_field_in_link": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "title_field": "full_name" } \ No newline at end of file diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index a8bb7a57d0..813fb5f3c2 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -70,7 +70,7 @@ "idx": 1, "in_create": 1, "links": [], - "modified": "2023-08-23 14:20:15.343339", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Core", "name": "Error Log", @@ -89,7 +89,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "title_field": "method" } \ No newline at end of file diff --git a/frappe/core/doctype/file/file.json b/frappe/core/doctype/file/file.json index 0477d82383..215178d8ec 100644 --- a/frappe/core/doctype/file/file.json +++ b/frappe/core/doctype/file/file.json @@ -189,7 +189,7 @@ "icon": "fa fa-file", "idx": 1, "links": [], - "modified": "2023-08-02 09:43:51.178012", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Core", "name": "File", @@ -217,7 +217,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "title_field": "file_name", "track_changes": 1 diff --git a/frappe/core/doctype/module_def/module_def.json b/frappe/core/doctype/module_def/module_def.json index 12830c8b4f..e9a03bce39 100644 --- a/frappe/core/doctype/module_def/module_def.json +++ b/frappe/core/doctype/module_def/module_def.json @@ -123,7 +123,7 @@ "link_fieldname": "module" } ], - "modified": "2022-01-03 13:56:52.817954", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Core", "name": "Module Def", @@ -160,7 +160,7 @@ ], "show_name_in_global_search": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/page/page.json b/frappe/core/doctype/page/page.json index b5e9941a6d..83117d802d 100644 --- a/frappe/core/doctype/page/page.json +++ b/frappe/core/doctype/page/page.json @@ -102,7 +102,7 @@ "icon": "fa fa-file", "idx": 1, "links": [], - "modified": "2023-10-22 22:41:25.568952", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Core", "name": "Page", @@ -129,7 +129,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/role/role.json b/frappe/core/doctype/role/role.json index 2039d3889d..1b147e3ddb 100644 --- a/frappe/core/doctype/role/role.json +++ b/frappe/core/doctype/role/role.json @@ -148,7 +148,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-08-05 18:33:27.694065", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Core", "name": "Role", @@ -169,7 +169,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1, "translated_doctype": 1 diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 45fa621cec..8f6ee3ca94 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -639,7 +639,7 @@ "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2023-11-27 14:08:01.927794", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Core", "name": "System Settings", @@ -655,7 +655,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/version/version.json b/frappe/core/doctype/version/version.json index 13c82fa2b2..570b53623c 100644 --- a/frappe/core/doctype/version/version.json +++ b/frappe/core/doctype/version/version.json @@ -54,7 +54,7 @@ "idx": 1, "in_create": 1, "links": [], - "modified": "2022-08-03 12:20:53.929691", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Core", "name": "Version", @@ -74,7 +74,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "title_field": "docname", "track_changes": 1 diff --git a/frappe/custom/doctype/client_script/client_script.json b/frappe/custom/doctype/client_script/client_script.json index 1db4dfe160..dddf0c0a04 100644 --- a/frappe/custom/doctype/client_script/client_script.json +++ b/frappe/custom/doctype/client_script/client_script.json @@ -77,7 +77,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-04-12 12:48:15.717985", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Custom", "name": "Client Script", @@ -108,7 +108,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index 332969b036..3f51636d9b 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -457,7 +457,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2023-10-25 06:55:10.713382", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field", @@ -488,7 +488,7 @@ ], "search_fields": "dt,label,fieldtype,options", "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.json b/frappe/custom/doctype/customize_form_field/customize_form_field.json index a3aec328bd..67c6c8ba95 100644 --- a/frappe/custom/doctype/customize_form_field/customize_form_field.json +++ b/frappe/custom/doctype/customize_form_field/customize_form_field.json @@ -483,7 +483,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2023-11-07 13:17:21.373626", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form Field", @@ -491,6 +491,6 @@ "owner": "Administrator", "permissions": [], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [] } \ No newline at end of file diff --git a/frappe/desk/doctype/note/note.json b/frappe/desk/doctype/note/note.json index 16b70171f5..4d60393d3f 100644 --- a/frappe/desk/doctype/note/note.json +++ b/frappe/desk/doctype/note/note.json @@ -86,7 +86,7 @@ "icon": "fa fa-file-text", "idx": 1, "links": [], - "modified": "2023-08-28 20:23:59.424943", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Desk", "name": "Note", @@ -141,7 +141,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "title_field": "title", "track_changes": 1 diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.json b/frappe/integrations/doctype/google_contacts/google_contacts.json index 2143cc6ca8..3858217a65 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.json +++ b/frappe/integrations/doctype/google_contacts/google_contacts.json @@ -99,7 +99,7 @@ } ], "links": [], - "modified": "2023-08-28 20:22:58.267442", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Integrations", "name": "Google Contacts", @@ -126,7 +126,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/integrations/doctype/google_drive/google_drive.json b/frappe/integrations/doctype/google_drive/google_drive.json index 592281be68..7bc967a6f8 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.json +++ b/frappe/integrations/doctype/google_drive/google_drive.json @@ -102,7 +102,7 @@ ], "issingle": 1, "links": [], - "modified": "2022-12-04 15:53:58.702389", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Integrations", "name": "Google Drive", @@ -120,7 +120,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/printing/doctype/letter_head/letter_head.json b/frappe/printing/doctype/letter_head/letter_head.json index daf1a20221..021f79ca93 100644 --- a/frappe/printing/doctype/letter_head/letter_head.json +++ b/frappe/printing/doctype/letter_head/letter_head.json @@ -168,7 +168,7 @@ "idx": 1, "links": [], "max_attachments": 3, - "modified": "2023-08-28 22:19:23.720332", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Printing", "name": "Letter Head", @@ -192,7 +192,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/social/doctype/review_level/review_level.json b/frappe/social/doctype/review_level/review_level.json index 06e3f397bc..bfc1618f2d 100644 --- a/frappe/social/doctype/review_level/review_level.json +++ b/frappe/social/doctype/review_level/review_level.json @@ -37,7 +37,7 @@ ], "istable": 1, "links": [], - "modified": "2022-08-03 12:20:51.571158", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Social", "name": "Review Level", @@ -45,7 +45,7 @@ "permissions": [], "quick_entry": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/website/doctype/web_page/web_page.json b/frappe/website/doctype/web_page/web_page.json index e7bd705272..2976285b1a 100644 --- a/frappe/website/doctype/web_page/web_page.json +++ b/frappe/website/doctype/web_page/web_page.json @@ -338,7 +338,7 @@ "index_web_pages_for_search": 1, "is_published_field": "published", "links": [], - "modified": "2022-03-09 01:45:28.548671", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Website", "name": "Web Page", @@ -357,7 +357,7 @@ "search_fields": "title", "show_name_in_global_search": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "title_field": "title", "track_changes": 1 diff --git a/frappe/website/doctype/website_settings/website_settings.json b/frappe/website/doctype/website_settings/website_settings.json index 7b9283c418..4707eef8df 100644 --- a/frappe/website/doctype/website_settings/website_settings.json +++ b/frappe/website/doctype/website_settings/website_settings.json @@ -476,7 +476,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2023-10-14 11:38:47.383840", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Website", "name": "Website Settings", @@ -498,7 +498,7 @@ } ], "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/workflow/doctype/workflow/workflow.json b/frappe/workflow/doctype/workflow/workflow.json index bddf8a66d7..435a2f56ad 100644 --- a/frappe/workflow/doctype/workflow/workflow.json +++ b/frappe/workflow/doctype/workflow/workflow.json @@ -104,7 +104,7 @@ "icon": "fa fa-random", "idx": 1, "links": [], - "modified": "2023-05-02 16:56:28.704844", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Workflow", "name": "Workflow", @@ -123,7 +123,7 @@ ], "show_name_in_global_search": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/workflow/doctype/workflow_state/workflow_state.json b/frappe/workflow/doctype/workflow_state/workflow_state.json index e622ae30c7..66e3ee27da 100644 --- a/frappe/workflow/doctype/workflow_state/workflow_state.json +++ b/frappe/workflow/doctype/workflow_state/workflow_state.json @@ -40,7 +40,7 @@ "icon": "fa fa-flag", "idx": 1, "links": [], - "modified": "2023-08-28 22:19:35.232574", + "modified": "2023-12-08 15:52:37.525003", "modified_by": "Administrator", "module": "Workflow", "name": "Workflow State", @@ -65,7 +65,7 @@ "quick_entry": 1, "show_name_in_global_search": 1, "sort_field": "modified", - "sort_order": "ASC", + "sort_order": "DESC", "states": [], "track_changes": 1, "translated_doctype": 1 From 0aea1de0f404f68c70448c08ec02fab1136c9676 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Wed, 13 Dec 2023 09:55:29 +0530 Subject: [PATCH 100/237] chore: remove testing internet connection for emails (#23581) --- .../doctype/email_account/email_account.py | 79 ++++++------------- 1 file changed, 24 insertions(+), 55 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index d7c75e03a1..6df8a4287c 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -238,9 +238,6 @@ class EmailAccount(Document): def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): """Returns logged in POP3/IMAP connection object.""" - if frappe.cache.get_value("workers:no-internet") == True: - return None - oauth_token = self.get_oauth_token() args = frappe._dict( { @@ -309,16 +306,13 @@ class EmailAccount(Document): except OSError: if in_receive: # timeout while connecting, see receive.py connect method - description = frappe.clear_last_message() if frappe.message_log else "Socket Error" - if test_internet(): - self.db_set("no_failed", self.no_failed + 1) - if self.no_failed > 2: - self.handle_incoming_connect_error(description=description) - else: - frappe.cache.set_value("workers:no-internet", True) - return None - else: - raise + description = frappe.message_log.pop() if frappe.message_log else "Socket Error" + self.db_set("no_failed", self.no_failed + 1) + if self.no_failed > 2: + self.handle_incoming_connect_error(description=description) + return + + raise @property def _password(self): @@ -495,29 +489,25 @@ class EmailAccount(Document): state.pop("_smtp_server_instance", None) def handle_incoming_connect_error(self, description): - if test_internet(): - if self.get_failed_attempts_count() > 2: - self.db_set("enable_incoming", 0) + if self.get_failed_attempts_count() > 2: + self.db_set("enable_incoming", 0) - for user in get_system_managers(only_name=True): - try: - assign_to.add( - { - "assign_to": user, - "doctype": self.doctype, - "name": self.name, - "description": description, - "priority": "High", - "notify": 1, - } - ) - except assign_to.DuplicateToDoError: - frappe.clear_last_message() - pass - else: - self.set_failed_attempts_count(self.get_failed_attempts_count() + 1) + for user in get_system_managers(only_name=True): + try: + assign_to.add( + { + "assign_to": user, + "doctype": self.doctype, + "name": self.name, + "description": description, + "priority": "High", + "notify": 1, + } + ) + except assign_to.DuplicateToDoError: + frappe.clear_last_message() else: - frappe.cache.set_value("workers:no-internet", True) + self.set_failed_attempts_count(self.get_failed_attempts_count() + 1) def set_failed_attempts_count(self, value): frappe.cache.set(f"{self.name}:email-account-failed-attempts", value) @@ -719,22 +709,6 @@ def get_append_to( return [[d] for d in set(email_append_to_list) if txt in d] -def test_internet(host="8.8.8.8", port=53, timeout=3): - """Returns True if internet is connected - - Host: 8.8.8.8 (google-public-dns-a.google.com) - OpenPort: 53/tcp - Service: domain (DNS/TCP) - """ - try: - socket.setdefaulttimeout(timeout) - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) - return True - except Exception as ex: - print(ex.message) - return False - - def notify_unreplied(): """Sends email notifications if there are unreplied Communications and `notify_if_unreplied` is set as true.""" @@ -792,11 +766,6 @@ def pull(now=False): """Will be called via scheduler, pull emails from all enabled Email accounts.""" from frappe.integrations.doctype.connected_app.connected_app import has_token - if frappe.cache.get_value("workers:no-internet") == True: - if test_internet(): - frappe.cache.set_value("workers:no-internet", False) - return - doctype = frappe.qb.DocType("Email Account") email_accounts = ( frappe.qb.from_(doctype) From 8e4b0ade78a3d290d70957a1614fbf8b8afac53d Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 13 Dec 2023 15:39:10 +0530 Subject: [PATCH 101/237] fix: remove web_form rate limit key (#23767) --- frappe/website/doctype/web_form/web_form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 2b2f3743f5..2c325742d1 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -444,7 +444,7 @@ def get_web_form_module(doc): @frappe.whitelist(allow_guest=True) -@rate_limit(key="web_form", limit=5, seconds=60) +@rate_limit(limit=5, seconds=60) def accept(web_form, data): """Save the web form""" data = frappe._dict(json.loads(data)) From 957f9a01b51feefb959c927360a898a1aa878622 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Dec 2023 15:57:18 +0530 Subject: [PATCH 102/237] fix: discard deleted doctypes unconditionally (#23770) --- frappe/website/doctype/portal_settings/portal_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/portal_settings/portal_settings.py b/frappe/website/doctype/portal_settings/portal_settings.py index b983548062..90a54f9c31 100644 --- a/frappe/website/doctype/portal_settings/portal_settings.py +++ b/frappe/website/doctype/portal_settings/portal_settings.py @@ -48,8 +48,8 @@ class PortalSettings(Document): if self.add_item(item): dirty = True + self.remove_deleted_doctype_items() if dirty: - self.remove_deleted_doctype_items() self.save() def on_update(self): From 119a4bdbc13a37745c44583aec86edfc9bd9ffc1 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Dec 2023 16:16:27 +0530 Subject: [PATCH 103/237] fix: remove delay from numeric field selection (#23759) --- frappe/public/js/frappe/form/controls/int.js | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/int.js b/frappe/public/js/frappe/form/controls/int.js index ffd3fe5fc3..122b43b498 100644 --- a/frappe/public/js/frappe/form/controls/int.js +++ b/frappe/public/js/frappe/form/controls/int.js @@ -2,21 +2,13 @@ frappe.ui.form.ControlInt = class ControlInt extends frappe.ui.form.ControlData static trigger_change_on_input_event = false; make() { super.make(); - // $(this.label_area).addClass('pull-right'); - // $(this.disp_area).addClass('text-right'); } make_input() { - var me = this; super.make_input(); - this.$input - // .addClass("text-right") - .on("focus", function () { - setTimeout(function () { - if (!document.activeElement) return; - document.activeElement.select(); - }, 100); - return false; - }); + this.$input.on("focus", () => { + document.activeElement?.select?.(); + return false; + }); } validate(value) { return this.parse(value); From 174e24f1597e6c3834a6888c2e7d2a7cabe69065 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Dec 2023 17:55:05 +0530 Subject: [PATCH 104/237] fix: validate custom portal menu entries during sync --- frappe/website/doctype/portal_settings/portal_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/portal_settings/portal_settings.py b/frappe/website/doctype/portal_settings/portal_settings.py index 90a54f9c31..bb37de2f01 100644 --- a/frappe/website/doctype/portal_settings/portal_settings.py +++ b/frappe/website/doctype/portal_settings/portal_settings.py @@ -69,6 +69,6 @@ class PortalSettings(Document): def remove_deleted_doctype_items(self): existing_doctypes = set(frappe.get_list("DocType", pluck="name")) - for menu_item in list(self.get("menu")): + for menu_item in list(self.get("menu") + self.get("custom_menu")): if menu_item.reference_doctype not in existing_doctypes: self.remove(menu_item) From 9ec54feff94e11e2fa397b26e360465e78c96184 Mon Sep 17 00:00:00 2001 From: Gursheen Kaur Anand <40693548+GursheenK@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:03:59 +0530 Subject: [PATCH 105/237] fix: skip using frm object for routing to audit trail (#23603) [skip ci[ --- frappe/public/js/frappe/form/form.js | 4 ---- frappe/public/js/frappe/form/toolbar.js | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 46c17343f8..044af703e1 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1276,10 +1276,6 @@ frappe.ui.form.Form = class FrappeForm { frappe.set_route("print", this.doctype, this.doc.name); } - show_audit_trail() { - frappe.set_route("audit-trail"); - } - navigate_records(prev) { let filters, sort_field, sort_order; let list_view = frappe.get_list_view(this.doctype); diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js index 443b608839..a1a4893061 100644 --- a/frappe/public/js/frappe/form/toolbar.js +++ b/frappe/public/js/frappe/form/toolbar.js @@ -506,7 +506,7 @@ frappe.ui.form.Toolbar = class Toolbar { this.page.add_menu_item( __("View Audit Trail"), function () { - me.frm.show_audit_trail(); + frappe.set_route("audit-trail"); }, true ); From cd7be151f791336e1543819282ddb6df7b81e449 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 14 Dec 2023 15:22:26 +0530 Subject: [PATCH 106/237] fix(rq_job): resolve deprecation warning (#23784) env/lib/python3.11/site-packages/rq/job.py:796: DeprecationWarning: job.exc_info is deprecated, use job.latest_result() instead. warnings.warn("job.exc_info is deprecated, use job.latest_result() instead.", DeprecationWarning) Signed-off-by: Akhil Narang --- frappe/core/doctype/rq_job/rq_job.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/rq_job/rq_job.py b/frappe/core/doctype/rq_job/rq_job.py index 81fa3fdf3e..69027af76f 100644 --- a/frappe/core/doctype/rq_job/rq_job.py +++ b/frappe/core/doctype/rq_job/rq_job.py @@ -152,6 +152,12 @@ def serialize_job(job: Job) -> frappe._dict: if matches := re.match(r".*) at 0x.*>", job_name): job_name = matches.group("func_name") + exc_info = None + + # Get exc_string from the job result if it exists + if job_result := job.latest_result(): + exc_info = job_result.exc_string + return frappe._dict( name=job.id, job_id=job.id, @@ -161,7 +167,7 @@ def serialize_job(job: Job) -> frappe._dict: started_at=convert_utc_to_system_timezone(job.started_at) if job.started_at else "", ended_at=convert_utc_to_system_timezone(job.ended_at) if job.ended_at else "", time_taken=(job.ended_at - job.started_at).total_seconds() if job.ended_at else "", - exc_info=job.exc_info, + exc_info=exc_info, arguments=frappe.as_json(job.kwargs), timeout=job.timeout, creation=convert_utc_to_system_timezone(job.created_at), From 687752359d81f9352e606a78fce7de76eeaf2065 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 14 Dec 2023 20:31:19 +0530 Subject: [PATCH 107/237] perf: Primary key is never nullable (#23788) People ususally write queries like these... ``` frappe.get_all(doctype, {"name": ("in", list_of_docs)) ``` Ocassionally, the `list_of_docs` is empty because it's dynamically generated and in this case we end up doing full table scan to find... nothing! --- frappe/model/db_query.py | 4 ++-- frappe/tests/test_db_query.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 7baa5848ed..70ecedd3c8 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -739,7 +739,7 @@ class DatabaseQuery: df = meta.get("fields", {"fieldname": f.fieldname}) df = df[0] if df else None - can_be_null = True + can_be_null = f.fieldname != "name" # primary key is never nullable value = None @@ -794,7 +794,7 @@ class DatabaseQuery: # if values contain '' or falsy values then only coalesce column # for `in` query this is only required if values contain '' or values are empty. # for `not in` queries we can't be sure as column values might contain null. - can_be_null = not getattr(df, "not_nullable", False) + can_be_null &= not getattr(df, "not_nullable", False) if f.operator.lower() == "in": can_be_null &= not f.value or any(v is None or v == "" for v in f.value) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index c5fdb8e10c..1ef4a23f65 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -1017,13 +1017,17 @@ class TestDBQuery(FrappeTestCase): self.assertEqual(call_args["order_by"], DefaultOrderBy) def test_coalesce_with_in_ops(self): - self.assertNotIn("ifnull", frappe.get_all("User", {"name": ("in", ["a", "b"])}, run=0)) - self.assertIn("ifnull", frappe.get_all("User", {"name": ("in", ["a", None])}, run=0)) - self.assertIn("ifnull", frappe.get_all("User", {"name": ("in", ["a", ""])}, run=0)) - self.assertIn("ifnull", frappe.get_all("User", {"name": ("in", [])}, run=0)) - self.assertIn("ifnull", frappe.get_all("User", {"name": ("not in", ["a"])}, run=0)) - self.assertIn("ifnull", frappe.get_all("User", {"name": ("not in", [])}, run=0)) - self.assertIn("ifnull", frappe.get_all("User", {"name": ("not in", [""])}, run=0)) + self.assertNotIn("ifnull", frappe.get_all("User", {"first_name": ("in", ["a", "b"])}, run=0)) + self.assertIn("ifnull", frappe.get_all("User", {"first_name": ("in", ["a", None])}, run=0)) + self.assertIn("ifnull", frappe.get_all("User", {"first_name": ("in", ["a", ""])}, run=0)) + self.assertIn("ifnull", frappe.get_all("User", {"first_name": ("in", [])}, run=0)) + self.assertIn("ifnull", frappe.get_all("User", {"first_name": ("not in", ["a"])}, run=0)) + self.assertIn("ifnull", frappe.get_all("User", {"first_name": ("not in", [])}, run=0)) + self.assertIn("ifnull", frappe.get_all("User", {"first_name": ("not in", [""])}, run=0)) + + # primary key is never nullable + self.assertNotIn("ifnull", frappe.get_all("User", {"name": ("in", ["a", None])}, run=0)) + self.assertNotIn("ifnull", frappe.get_all("User", {"name": ("in", ["a", ""])}, run=0)) def test_ambiguous_linked_tables(self): from frappe.desk.reportview import get From b7c2989823f527ce958d0bfcd672928c9642843e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 14 Dec 2023 20:46:49 +0530 Subject: [PATCH 108/237] feat: `add-database-index` command to add and persist custom indexes (#23787) --- frappe/commands/site.py | 35 +++++++++++++++++++++++++++++++++++ frappe/tests/test_commands.py | 11 +++++++++++ 2 files changed, 46 insertions(+) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9225f31ccb..67d3f7820c 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -524,6 +524,40 @@ def list_apps(context, format): click.echo(frappe.as_json(summary_dict)) +@click.command("add-database-index") +@click.option("--doctype", help="DocType on which index needs to be added") +@click.option( + "--column", + multiple=True, + help="Column to index. Multiple columns will create multi-column index in given order. To create a multiple, single column index, execute the command multiple times.", +) +@pass_context +def add_db_index(context, doctype, column): + "Adds a new DB index and creates a property setter to persist it." + from frappe.custom.doctype.property_setter.property_setter import make_property_setter + + columns = column # correct naming + for site in context.sites: + frappe.connect(site=site) + try: + frappe.db.add_index(doctype, columns) + if len(columns) == 1: + make_property_setter( + doctype, + columns[0], + property="search_index", + value="1", + property_type="Check", + for_doctype=False, # Applied on docfield + ) + frappe.db.commit() + finally: + frappe.destroy() + + if not context.sites: + raise SiteNotSpecifiedError + + @click.command("add-system-manager") @click.argument("email") @click.option("--first-name") @@ -1436,6 +1470,7 @@ def add_new_user( commands = [ add_system_manager, add_user_for_sites, + add_db_index, backup, drop_site, install_app, diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 7ca14e595c..3b3d6f1268 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -788,6 +788,17 @@ class TestBenchBuild(BaseTestCommands): ) +class TestDBUtils(BaseTestCommands): + def test_db_add_index(self): + field = "reset_password_key" + self.execute("bench --site {site} add-database-index --doctype User --column " + field, {}) + frappe.db.rollback() + index_name = frappe.db.get_index_name((field,)) + self.assertTrue(frappe.db.has_index("tabUser", index_name)) + meta = frappe.get_meta("User", cached=False) + self.assertTrue(meta.get_field(field).search_index) + + class TestSchedulerUtils(BaseTestCommands): # Retry just in case there are stuck queued jobs @retry( From eb5448209e6459e0d79cfb5c30f91924cbcc8331 Mon Sep 17 00:00:00 2001 From: Niraj Gautam Date: Thu, 14 Dec 2023 21:10:47 +0530 Subject: [PATCH 109/237] fix(jinja): Update error message in validate_template function (#23783) [skip ci] --- frappe/utils/jinja.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index 37a3916535..694887395e 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -66,8 +66,7 @@ def validate_template(html): try: jenv.from_string(html) except TemplateSyntaxError as e: - frappe.msgprint(f"Line {e.lineno}: {e.message}") - frappe.throw(frappe._("Syntax error in template")) + frappe.throw(frappe._(f"Syntax error in template as line {e.lineno}: {e.message}")) def render_template(template, context=None, is_path=None, safe_render=True): From 87e0ccd555f9f9085dc1d6bc7c15ad98b277871c Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 15 Dec 2023 11:31:25 +0530 Subject: [PATCH 110/237] fix: load 100 records for larger screen --- frappe/public/js/frappe/list/base_list.js | 2 +- frappe/public/js/frappe/utils/common.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index a9659d079f..0aaa1f1e31 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -44,7 +44,7 @@ frappe.views.BaseList = class BaseList { this.user_settings = frappe.get_user_settings(this.doctype); this.start = 0; - this.page_length = 20; + this.page_length = frappe.is_large_screen() ? 100 : 20; this.data = []; this.method = "frappe.desk.reportview.get"; diff --git a/frappe/public/js/frappe/utils/common.js b/frappe/public/js/frappe/utils/common.js index 870d5b7182..7a8e256c23 100644 --- a/frappe/public/js/frappe/utils/common.js +++ b/frappe/public/js/frappe/utils/common.js @@ -274,6 +274,10 @@ frappe.is_mobile = function () { return $(document).width() < 768; }; +frappe.is_large_screen = function () { + return $(document).width() > 1920; +}; + frappe.utils.xss_sanitise = function (string, options) { // Reference - https://www.owasp.org/index.php/XSS_(Cross_Site_Scripting)_Prevention_Cheat_Sheet let sanitised = string; // un-sanitised string. From 2fa1120b0f4e483a54da7c77af9822cfff3c3078 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 15 Dec 2023 11:45:14 +0530 Subject: [PATCH 111/237] fix: consider height istead of width --- frappe/public/js/frappe/utils/common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/common.js b/frappe/public/js/frappe/utils/common.js index 7a8e256c23..10e97a3ffd 100644 --- a/frappe/public/js/frappe/utils/common.js +++ b/frappe/public/js/frappe/utils/common.js @@ -275,7 +275,7 @@ frappe.is_mobile = function () { }; frappe.is_large_screen = function () { - return $(document).width() > 1920; + return $(document).height() > 1180; }; frappe.utils.xss_sanitise = function (string, options) { From d39ecd3932cf1bd93acbdaf5eedd11175bd92d68 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 15 Dec 2023 12:51:29 +0530 Subject: [PATCH 112/237] fix: Consider user and sender both for "system user" (#23806) Email is "sender" which might not be linked with any user. --- frappe/core/doctype/communication/communication.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 32500c9158..1422c446cd 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -662,7 +662,10 @@ def update_parent_document_on_communication(doc): def update_first_response_time(parent, communication): if parent.meta.has_field("first_response_time") and not parent.get("first_response_time"): - if is_system_user(communication.sender): + if ( + is_system_user(communication.sender) + or frappe.get_cached_value("User", frappe.session.user, "user_type") == "System User" + ): if communication.sent_or_received == "Sent": first_responded_on = communication.creation if parent.meta.has_field("first_responded_on"): From 4a7c60fb5a98197e11517aa0c02ac3e6f589765a Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Fri, 15 Dec 2023 17:02:09 +0530 Subject: [PATCH 113/237] refactor: use simple condition for filter values --- frappe/desk/query_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 59c9114ec9..bc12f52955 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -394,7 +394,7 @@ def build_xlsx_data( filter_data = [] filters = data.filters for filter_name, filter_value in filters.items(): - if filter_value in ["", None, []]: + if not filter_value: continue filter_value = ( ", ".join(map(lambda x: cstr(x), filter_value)) From 801b9e1140f19b2a5ec6f00549c091fc661303ce Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Fri, 15 Dec 2023 17:51:22 +0530 Subject: [PATCH 114/237] fix: labels for boolean filters --- frappe/public/js/frappe/views/reports/query_report.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 72060d8b1c..5ac979f029 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1512,10 +1512,13 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { filters ); } + let boolean_labels = { 1: __("Yes"), 0: __("No") }; let applied_filters = Object.fromEntries( Object.entries(filters).map(([key, value]) => [ frappe.query_report.get_filter(key).df.label, - value, + frappe.query_report.get_filter(key).df.fieldtype == "Check" + ? boolean_labels[value] + : value, ]) ); From 40e48c9ac4696b5f0f387535c461a779224c5386 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 15 Dec 2023 17:53:11 +0530 Subject: [PATCH 115/237] feat: `describe-database-table` to get stats about a table (#23813) * feat: `describe-database-table` to get stats about a table This provides description of table, indexes, total row count in machine parseable format. Towards https://github.com/frappe/press/issues/1265 * Update test_commands.py --- frappe/commands/site.py | 88 +++++++++++++++++++++++++++++++++++ frappe/tests/test_commands.py | 9 ++++ 2 files changed, 97 insertions(+) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 67d3f7820c..31a4be3340 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -558,6 +558,93 @@ def add_db_index(context, doctype, column): raise SiteNotSpecifiedError +@click.command("describe-database-table") +@click.option("--doctype", help="DocType to describe") +@click.option( + "--column", + multiple=True, + help="Explicitly fetch accurate cardinality from table data. This can be quite slow on large tables.", +) +@pass_context +def describe_database_table(context, doctype, column): + """Describes various statistics about the table. + + This is useful to build integration like + This includes: + 1. Schema + 2. Indexes + 3. stats - total count of records + 4. if column is specified then extra stats are generated for column: + Distinct values count in column + """ + import json + + for site in context.sites: + frappe.connect(site=site) + try: + data = _extract_table_stats(doctype, column) + # NOTE: Do not print anything else in this to avoid clobbering the output. + print(json.dumps(data, indent=2)) + finally: + frappe.destroy() + + if not context.sites: + raise SiteNotSpecifiedError + + +def _extract_table_stats(doctype: str, columns: list[str]) -> dict: + from frappe.utils import get_table_name + + table = get_table_name(doctype, wrap_in_backticks=True) + + schema = [] + for field in frappe.db.sql(f"describe {table}", as_dict=True): + schema.append( + { + "column": field["Field"], + "type": field["Type"], + "is_nullable": field["Null"], + "default": field["Default"], + } + ) + + def update_cardinality(column, value): + for col in schema: + if col["column"] == column: + col["cardinality"] = value + break + + indexes = [] + for idx in frappe.db.sql(f"show index from {table}", as_dict=True): + indexes.append( + { + "unique": not idx["Non_unique"], + "cardinality": idx["Cardinality"], + "name": idx["Key_name"], + "sequence": idx["Seq_in_index"], + "nullable": idx["Null"], + "column": idx["Column_name"], + "type": idx["Index_type"], + } + ) + if idx["Seq_in_index"] == 1: + update_cardinality(idx["Column_name"], idx["Cardinality"]) + + total_rows = frappe.db.count(doctype) + + # fetch accurate cardinality for columns by query. WARN: This can take a lot of time. + for column in columns: + cardinality = frappe.db.sql(f"select count(distinct {column}) from {table}")[0][0] + update_cardinality(column, cardinality) + + return { + "table_name": table.strip("`"), + "total_rows": total_rows, + "schema": schema, + "indexes": indexes, + } + + @click.command("add-system-manager") @click.argument("email") @click.option("--first-name") @@ -1471,6 +1558,7 @@ commands = [ add_system_manager, add_user_for_sites, add_db_index, + describe_database_table, backup, drop_site, install_app, diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 3b3d6f1268..3cff14afa9 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -798,6 +798,15 @@ class TestDBUtils(BaseTestCommands): meta = frappe.get_meta("User", cached=False) self.assertTrue(meta.get_field(field).search_index) + @run_only_if(db_type_is.MARIADB) + def test_describe_table(self): + self.execute("bench --site {site} describe-database-table --doctype User", {}) + self.assertIn("user_type", self.stdout) + + # Ensure that output is machine parseable + stats = json.loads(self.stdout) + self.assertIn("total_rows", stats) + class TestSchedulerUtils(BaseTestCommands): # Retry just in case there are stuck queued jobs From c38e842646021d74106e984b3a748e1d8f1b9121 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 16 Dec 2023 13:48:46 +0530 Subject: [PATCH 116/237] perf(migrate): Only re-evaluate users which are potentially changed (#23820) When roles are added in fixture `desk_access` always appears to be "changed" because of how fixtures work. This causes all users with the said role to be re-evaluated. This is unnecessary computation because desk_access rarely changes in most apps. Fix: See if role's desk access is same as user's desk access and don't re-evalute them. E.g. If role fixture without desk access is being migrated then it will skip all users who are already website users. Likewise role with desk access will skip all users who are already system users. --- frappe/core/doctype/role/role.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index b3ec48d946..87ff615e0f 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -80,12 +80,23 @@ class Role(Document): if frappe.flags.in_install: return if self.has_value_changed("desk_access"): - for user_name in get_users(self.name): - user = frappe.get_doc("User", user_name) - user_type = user.user_type - user.set_system_user() - if user_type != user.user_type: - user.save() + self.update_user_type_on_change() + + def update_user_type_on_change(self): + """When desk access changes, all the users that have this role need to be re-evaluated""" + + users_with_role = get_users(self.name) + + # perf: Do not re-evaluate users who already have same desk access that this role permits. + role_user_type = "System User" if self.desk_access else "Website User" + users_with_same_user_type = frappe.get_all("User", {"user_type": role_user_type}, pluck="name") + + for user_name in set(users_with_role) - set(users_with_same_user_type): + user = frappe.get_doc("User", user_name) + user_type = user.user_type + user.set_system_user() + if user_type != user.user_type: + user.save() def get_info_based_on_role(role, field="email", ignore_permissions=False): From 3adc83e7c6960f9c57927889f180e980d2ff42aa Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 16 Dec 2023 13:59:26 +0530 Subject: [PATCH 117/237] perf(migrate): update only affected users via role profile (#23821) This is same as https://github.com/frappe/frappe/pull/23820 but for role profile --- .../core/doctype/role_profile/role_profile.py | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/role_profile/role_profile.py b/frappe/core/doctype/role_profile/role_profile.py index b005a695a4..74c34e3993 100644 --- a/frappe/core/doctype/role_profile/role_profile.py +++ b/frappe/core/doctype/role_profile/role_profile.py @@ -1,6 +1,8 @@ # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE +from collections import defaultdict + import frappe from frappe.model.document import Document @@ -24,9 +26,24 @@ class RoleProfile(Document): def on_update(self): """Changes in role_profile reflected across all its user""" - users = frappe.get_all("User", filters={"role_profile_name": self.name}) - roles = [role.role for role in self.roles] - for d in users: - user = frappe.get_doc("User", d) - user.set("roles", []) - user.add_roles(*roles) + has_role = frappe.qb.DocType("Has Role") + user = frappe.qb.DocType("User") + + all_current_roles = ( + frappe.qb.from_(user) + .join(has_role) + .on(user.name == has_role.parent) + .where(user.role_profile_name == self.name) + .select(user.name, has_role.role) + ).run() + + user_roles = defaultdict(set) + for user, role in all_current_roles: + user_roles[user].add(role) + + role_profile_roles = {role.role for role in self.roles} + for user, roles in user_roles.items(): + if roles != role_profile_roles: + user = frappe.get_doc("User", user) + user.roles = [] + user.add_roles(*role_profile_roles) From d5ccb512512cf050bd0c7526c78b5e5469f38089 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 14:25:39 +0530 Subject: [PATCH 118/237] docs: frappe.as_json --- frappe/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/__init__.py b/frappe/__init__.py index b59064e95d..b781f46c64 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -2011,6 +2011,7 @@ def get_value(*args, **kwargs): def as_json(obj: dict | list, indent=1, separators=None, ensure_ascii=True) -> str: + """Returns the JSON string representation of the given `obj`.""" from frappe.utils.response import json_handler if separators is None: From 0fa0d1ff0e5faf1e3ff4ad9c00f3a36ebec03abe Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 16:48:11 +0530 Subject: [PATCH 119/237] docs: frappe.bold --- frappe/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index b781f46c64..a504d824c1 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -2297,7 +2297,8 @@ def get_desk_link(doctype, name): return html.format(doctype=doctype, name=name, doctype_local=_(doctype)) -def bold(text): +def bold(text: str) -> str: + """Returns `text` wrapped in `` tags.""" return f"{text}" From 49a46632282011a2cc1d57389105a99b049b4909 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 16:49:35 +0530 Subject: [PATCH 120/237] docs: html2text --- frappe/core/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/utils.py b/frappe/core/utils.py index 5f388f5458..93e09ce695 100644 --- a/frappe/core/utils.py +++ b/frappe/core/utils.py @@ -86,6 +86,7 @@ def ljust_list(_list, length, fill_word=None): return _list -def html2text(html, strip_links=False, wrap=True): +def html2text(html: str, strip_links=False, wrap=True) -> str: + """Returns the given `html` as markdown text.""" strip = ["a"] if strip_links else None return md(html, heading_style="ATX", strip=strip, wrap=wrap) From 51a03c52f1a8390b0b630ccdc22d51d944e49c0f Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:13:55 +0530 Subject: [PATCH 121/237] docs: safe_exec get_hooks --- frappe/utils/safe_exec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index b4581d8387..d8b30b871d 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -398,7 +398,13 @@ def get_python_builtins(): } -def get_hooks(hook=None, default=None, app_name=None): +def get_hooks(hook: str = None, default=None, app_name: str = None) -> frappe._dict: + """Get hooks via `app/hooks.py` + + :param hook: Name of the hook. Will gather all hooks for this name and return as a list. + :param default: Default if no hook found. + :param app_name: Filter by app.""" + hooks = frappe.get_hooks(hook=hook, default=default, app_name=app_name) return copy.deepcopy(hooks) From 1a2d8fe78aa8a5a5de018c3addf729d00fce0a40 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:18:31 +0530 Subject: [PATCH 122/237] docs: get_system_settings * IMO this should be renamed to something like `get_from_system_settings` --- frappe/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index a504d824c1..036cc371c3 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -2321,7 +2321,8 @@ def get_website_settings(key): return local.website_settings.get(key) -def get_system_settings(key): +def get_system_settings(key: str): + """Get the value associated with the given `key` from System Settings DocType.""" if not hasattr(local, "system_settings"): try: local.system_settings = get_cached_doc("System Settings") From d16683a699da1b0969d781b8dff7b1e3592d833f Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:21:44 +0530 Subject: [PATCH 123/237] docs: is_invalid_data_string --- frappe/utils/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 58a161b8e6..37f28500a4 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -71,6 +71,7 @@ def get_start_of_week_index() -> int: def is_invalid_date_string(date_string: str) -> bool: + """Returns True if the date string is invalid or None or empty.""" # dateutil parser does not agree with dates like "0001-01-01" or "0000-00-00" return not isinstance(date_string, str) or ( (not date_string) or (date_string or "").startswith(("0001-01-01", "0000-00-00")) From 015627464b031f2939f20cb0b6e4b8282d93d80a Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:35:25 +0530 Subject: [PATCH 124/237] docs: get_datetime and to_timedelta --- frappe/utils/data.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 37f28500a4..0177293ef3 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -107,6 +107,16 @@ def getdate( def get_datetime( datetime_str: Optional["DateTimeLikeObject"] = None, ) -> datetime.datetime | None: + """Returns the below mentioned values based on the given `datetime_str`: + + * If `datetime_str` is None, returns datetime object of current datetime + * If `datetime_str` is already a datetime object, returns the same + * If `datetime_str` is a timedelta object, returns the same + * If `datetime_str` is a list or tuple, returns a datetime object + * If `datetime_str` is a date object, returns a datetime object + * If `datetime_str` is a valid date string, returns a datetime object for the same + * If `datetime_str` is an invalid date string, returns None + """ if datetime_str is None: return now_datetime() @@ -159,6 +169,9 @@ def get_timedelta(time: str | None = None) -> datetime.timedelta | None: def to_timedelta(time_str: str | datetime.time) -> datetime.timedelta: + """Returns a `datetime.timedelta` object from the given string or `datetime.time` object. + If the given argument is not a string or a `datetime.time` object, it is returned as is. + """ if isinstance(time_str, datetime.time): time_str = str(time_str) From 40b89566524a06fe1fe6215c23b63829aec56975 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:42:46 +0530 Subject: [PATCH 125/237] docs: add_days, add_months, add_years --- frappe/utils/data.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 0177293ef3..0df5a32ef5 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -275,15 +275,18 @@ def add_to_date( return date -def add_days(date, days): +def add_days(date: DateTimeLikeObject, days: NumericType) -> DateTimeLikeObject: + """Returns a new date after adding the given number of `days` to the given `date`.""" return add_to_date(date, days=days) -def add_months(date, months): +def add_months(date: DateTimeLikeObject, months: NumericType) -> DateTimeLikeObject: + """Returns a new date after adding the given number of `months` to the given `date`.""" return add_to_date(date, months=months) -def add_years(date, years): +def add_years(date: DateTimeLikeObject, years: NumericType) -> DateTimeLikeObject: + """Returns a new date after adding the given number of `years` to the given `date`.""" return add_to_date(date, years=years) From d1131db2a762c62b2611862b01010538d134dd3b Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:48:25 +0530 Subject: [PATCH 126/237] docs: days diff + new days_diff util function --- frappe/utils/data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 0df5a32ef5..f3100cbbf5 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -291,6 +291,12 @@ def add_years(date: DateTimeLikeObject, years: NumericType) -> DateTimeLikeObjec def date_diff(string_ed_date, string_st_date): + """Returns the difference between given two dates in days.""" + return days_diff(string_ed_date, string_st_date) + + +def days_diff(string_ed_date, string_st_date): + """Returns the difference between given two dates in days.""" return (getdate(string_ed_date) - getdate(string_st_date)).days From 9be9da49b346a3707b32d8fdcab1fe86f30da496 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:49:54 +0530 Subject: [PATCH 127/237] fix: add missing type hints --- frappe/utils/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index f3100cbbf5..f54ffe364e 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -290,12 +290,12 @@ def add_years(date: DateTimeLikeObject, years: NumericType) -> DateTimeLikeObjec return add_to_date(date, years=years) -def date_diff(string_ed_date, string_st_date): +def date_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject) -> int: """Returns the difference between given two dates in days.""" return days_diff(string_ed_date, string_st_date) -def days_diff(string_ed_date, string_st_date): +def days_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject) -> int: """Returns the difference between given two dates in days.""" return (getdate(string_ed_date) - getdate(string_st_date)).days From 394af95c31772acff8d77c8527b6f18fe3315719 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:53:10 +0530 Subject: [PATCH 128/237] docs: month_diff and time_diff --- frappe/utils/data.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index f54ffe364e..c817eeb2c7 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -300,13 +300,17 @@ def days_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeOb return (getdate(string_ed_date) - getdate(string_st_date)).days -def month_diff(string_ed_date, string_st_date): +def month_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject) -> int: + """Returns the difference between given two dates in months.""" ed_date = getdate(string_ed_date) st_date = getdate(string_st_date) return (ed_date.year - st_date.year) * 12 + ed_date.month - st_date.month + 1 -def time_diff(string_ed_date, string_st_date): +def time_diff( + string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject +) -> datetime.timedelta: + """Returns the difference between given two dates as `datetime.timedelta` object.""" return get_datetime(string_ed_date) - get_datetime(string_st_date) From 23b7f4c4ff61e48a8b7d442e577984b1505e50ad Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 22:57:40 +0530 Subject: [PATCH 129/237] docs: time_diff_in_seconds and time_diff_in_hours --- frappe/utils/data.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index c817eeb2c7..f438602837 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -314,11 +314,17 @@ def time_diff( return get_datetime(string_ed_date) - get_datetime(string_st_date) -def time_diff_in_seconds(string_ed_date, string_st_date): +def time_diff_in_seconds( + string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject +) -> float: + """Returns the difference between given two dates in seconds.""" return time_diff(string_ed_date, string_st_date).total_seconds() -def time_diff_in_hours(string_ed_date, string_st_date): +def time_diff_in_hours( + string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject +) -> float: + """Returns the difference between given two dates in hours.""" return round(float(time_diff(string_ed_date, string_st_date).total_seconds()) / 3600, 6) From 4992bdf93779723a128d1dadaa6a20c679483c8a Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 23:05:46 +0530 Subject: [PATCH 130/237] docs: now_datetime, get_timestamp --- frappe/utils/data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index f438602837..871a9ad5a1 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -328,12 +328,14 @@ def time_diff_in_hours( return round(float(time_diff(string_ed_date, string_st_date).total_seconds()) / 3600, 6) -def now_datetime(): +def now_datetime() -> datetime.datetime: + """Returns the current datetime in system timezone.""" dt = convert_utc_to_system_timezone(datetime.datetime.now(pytz.UTC)) return dt.replace(tzinfo=None) -def get_timestamp(date): +def get_timestamp(date: DateTimeLikeObject) -> float: + """Returns the Unix timestamp (seconds since Epoch) for the given `date`.""" return time.mktime(getdate(date).timetuple()) From a2fd4905b5daaeff8f895c3c4ac58adedc80fc09 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 23:08:09 +0530 Subject: [PATCH 131/237] refactor(get_timestamp): return current timestamp if date is none --- frappe/utils/data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 871a9ad5a1..250dc6fb44 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -334,8 +334,10 @@ def now_datetime() -> datetime.datetime: return dt.replace(tzinfo=None) -def get_timestamp(date: DateTimeLikeObject) -> float: - """Returns the Unix timestamp (seconds since Epoch) for the given `date`.""" +def get_timestamp(date: Optional["DateTimeLikeObject"] = None) -> float: + """Returns the Unix timestamp (seconds since Epoch) for the given `date`. + If `date` is None, the current timestamp is returned. + """ return time.mktime(getdate(date).timetuple()) From 318e3a14e9f44817350206d805b288c18ac91de8 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sat, 16 Dec 2023 23:10:34 +0530 Subject: [PATCH 132/237] docs: today --- frappe/utils/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 250dc6fb44..e9a53609ff 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -396,6 +396,7 @@ def nowdate() -> str: def today() -> str: + """Returns today's date in `yyyy-mm-dd` format.""" return nowdate() From d3cde6f7431cbf42032d883c1cb3c979a15ecbae Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 00:17:18 +0530 Subject: [PATCH 133/237] chore: add type hints to get_eta --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index e9a53609ff..eb48f29f44 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -341,7 +341,7 @@ def get_timestamp(date: Optional["DateTimeLikeObject"] = None) -> float: return time.mktime(getdate(date).timetuple()) -def get_eta(from_time, percent_complete): +def get_eta(from_time: DateTimeLikeObject, percent_complete) -> str: diff = time_diff(now_datetime(), from_time).total_seconds() return str(datetime.timedelta(seconds=(100 - percent_complete) / percent_complete * diff)) From ed36873af4f57389a557bb6bae7230e6bc527384 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 00:19:34 +0530 Subject: [PATCH 134/237] docs: get_system_timezone --- frappe/utils/data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index eb48f29f44..bab8c97ebe 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -350,7 +350,8 @@ def _get_system_timezone(): return frappe.get_system_settings("time_zone") or "Asia/Kolkata" # Default to India ?! -def get_system_timezone(): +def get_system_timezone() -> str: + """Returns the system timezone.""" if frappe.local.flags.in_test: return _get_system_timezone() From 8156d24ec99ab71f5205ea6b1ca41cbb884917e9 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 00:24:01 +0530 Subject: [PATCH 135/237] docs: get_abbr --- frappe/utils/data.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index bab8c97ebe..ea786e5153 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -402,6 +402,16 @@ def today() -> str: def get_abbr(string: str, max_len: int = 2) -> str: + """Returns the abbreviation of the given string. + + Examples: + + * "John Doe" => "JD" + * "Jenny Jane Doe" => "JJ" (default, `max_len` = 2) + * "Jenny Jane Doe" => "JJD" (`max_len` = 3) + + Returns "?" if the given string is empty. + """ abbr = "" for part in string.split(" "): if len(abbr) < max_len and part: From cf28fe2da49ff3bb8a12d784e25fed9db05b05dd Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 00:29:48 +0530 Subject: [PATCH 136/237] fix: consistency --- frappe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 036cc371c3..67cae7556d 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -2322,7 +2322,7 @@ def get_website_settings(key): def get_system_settings(key: str): - """Get the value associated with the given `key` from System Settings DocType.""" + """Returns the value associated with the given `key` from System Settings DocType.""" if not hasattr(local, "system_settings"): try: local.system_settings = get_cached_doc("System Settings") From 24c8b0fb6c4356ef16431f443c6e64b607b29c4a Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 11:59:27 +0530 Subject: [PATCH 137/237] docs: make_*_request utils --- frappe/integrations/utils.py | 62 ++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index 86f8b0b1ef..2cafc8ba61 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -10,7 +10,9 @@ from frappe import _ from frappe.utils import get_request_session -def make_request(method, url, auth=None, headers=None, data=None, json=None, params=None): +def make_request( + method: str, url: str, auth=None, headers=None, data=None, json=None, params=None +): auth = auth or "" data = data or {} headers = headers or {} @@ -31,23 +33,71 @@ def make_request(method, url, auth=None, headers=None, data=None, json=None, par raise exc -def make_get_request(url, **kwargs): +def make_get_request(url: str, **kwargs): + """Makes a 'GET' HTTP request to the given `url` and returns processed response. + + You can optionally pass the below parameters: + + * `headers`: Headers to be set in the request. + * `params`: Query parameters to be passed in the request. + * `auth`: Auth credentials. + """ return make_request("GET", url, **kwargs) -def make_post_request(url, **kwargs): +def make_post_request(url: str, **kwargs): + """Makes a 'POST' HTTP request to the given `url` and returns processed response. + + You can optionally pass the below parameters: + + * `headers`: Headers to be set in the request. + * `data`: Data to be passed in body of the request. + * `json`: JSON to be passed in the request. + * `params`: Query parameters to be passed in the request. + * `auth`: Auth credentials. + """ return make_request("POST", url, **kwargs) -def make_put_request(url, **kwargs): +def make_put_request(url: str, **kwargs): + """Makes a 'PUT' HTTP request to the given `url` and returns processed response. + + You can optionally pass the below parameters: + + * `headers`: Headers to be set in the request. + * `data`: Data to be passed in body of the request. + * `json`: JSON to be passed in the request. + * `params`: Query parameters to be passed in the request. + * `auth`: Auth credentials. + """ return make_request("PUT", url, **kwargs) -def make_patch_request(url, **kwargs): +def make_patch_request(url: str, **kwargs): + """Makes a 'PATCH' HTTP request to the given `url` and returns processed response. + + You can optionally pass the below parameters: + + * `headers`: Headers to be set in the request. + * `data`: Data to be passed in body of the request. + * `json`: JSON to be passed in the request. + * `params`: Query parameters to be passed in the request. + * `auth`: Auth credentials. + """ return make_request("PATCH", url, **kwargs) -def make_delete_request(url, **kwargs): +def make_delete_request(url: str, **kwargs): + """Makes a 'DELETE' HTTP request to the given `url` and returns processed response. + + You can optionally pass the below parameters: + + * `headers`: Headers to be set in the request. + * `data`: Data to be passed in body of the request. + * `json`: JSON to be passed in the request. + * `params`: Query parameters to be passed in the request. + * `auth`: Auth credentials. + """ return make_request("DELETE", url, **kwargs) From ca33deae12738dfac14ccd6fb1d1d41b142c6504 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 17 Dec 2023 12:07:14 +0530 Subject: [PATCH 138/237] fix: cast SQL booleans to python --- frappe/commands/site.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 31a4be3340..9b8d2d9131 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -593,7 +593,10 @@ def describe_database_table(context, doctype, column): def _extract_table_stats(doctype: str, columns: list[str]) -> dict: - from frappe.utils import get_table_name + from frappe.utils import cstr, get_table_name + + def sql_bool(val): + return cstr(val).lower() in ("yes", "1", "true") table = get_table_name(doctype, wrap_in_backticks=True) @@ -603,7 +606,7 @@ def _extract_table_stats(doctype: str, columns: list[str]) -> dict: { "column": field["Field"], "type": field["Type"], - "is_nullable": field["Null"], + "is_nullable": sql_bool(field["Null"]), "default": field["Default"], } ) @@ -618,11 +621,11 @@ def _extract_table_stats(doctype: str, columns: list[str]) -> dict: for idx in frappe.db.sql(f"show index from {table}", as_dict=True): indexes.append( { - "unique": not idx["Non_unique"], + "unique": not sql_bool(idx["Non_unique"]), "cardinality": idx["Cardinality"], "name": idx["Key_name"], "sequence": idx["Seq_in_index"], - "nullable": idx["Null"], + "nullable": sql_bool(idx["Null"]), "column": idx["Column_name"], "type": idx["Index_type"], } From 38fdf2d7cb0c643ec28dd992fa3513f869e4a5fb Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 12:47:39 +0530 Subject: [PATCH 139/237] docs: get_time --- frappe/utils/data.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index ea786e5153..cf5790a583 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -541,7 +541,13 @@ def get_year_ending(date) -> datetime.date: return add_to_date(next_year_start, days=-1) -def get_time(time_str: str) -> datetime.time: +def get_time( + time_str: str | datetime.datetime | datetime.time | datetime.timedelta, +) -> datetime.time: + """Returns a `datetime.time` object for the given `time_str`. + + If the given argument is already a `datetime.time` object, it is returned as is.""" + if isinstance(time_str, datetime.datetime): return time_str.time() elif isinstance(time_str, datetime.time): From 967755cb77bcf7ebb02f49ca58eba22b56413416 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 12:52:09 +0530 Subject: [PATCH 140/237] docs: nowtime --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index cf5790a583..2191d1004b 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -421,7 +421,7 @@ def get_abbr(string: str, max_len: int = 2) -> str: def nowtime() -> str: - """return current time in hh:mm""" + """Returns current time (system timezone) in `hh:mm:ss` format.""" return now_datetime().strftime(TIME_FORMAT) From b3d5db60fe55deeb4db25ed19f1fe7a2b9605482 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 13:17:05 +0530 Subject: [PATCH 141/237] docs: get_datetime_in_timezone and convert_utc_to_system_timezone --- frappe/utils/data.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 2191d1004b..fa4fb41af3 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -358,7 +358,7 @@ def get_system_timezone() -> str: return frappe.cache.get_value("time_zone", _get_system_timezone) -def convert_utc_to_timezone(utc_timestamp, time_zone): +def convert_utc_to_timezone(utc_timestamp: datetime.datetime, time_zone: str) -> datetime.datetime: from pytz import UnknownTimeZoneError, timezone if utc_timestamp.tzinfo is None: @@ -369,12 +369,14 @@ def convert_utc_to_timezone(utc_timestamp, time_zone): return utc_timestamp -def get_datetime_in_timezone(time_zone): +def get_datetime_in_timezone(time_zone: str) -> datetime.datetime: + """Returns the current datetime in the given timezone (e.g. 'Asia/Kolkata').""" utc_timestamp = datetime.datetime.now(pytz.UTC) return convert_utc_to_timezone(utc_timestamp, time_zone) -def convert_utc_to_system_timezone(utc_timestamp): +def convert_utc_to_system_timezone(utc_timestamp: datetime.datetime) -> datetime.datetime: + """Returns the given UTC `datetime` timestamp in system timezone.""" time_zone = get_system_timezone() return convert_utc_to_timezone(utc_timestamp, time_zone) @@ -494,7 +496,8 @@ def get_normalized_weekday_index(dt): return (dt.weekday() + 1) % 7 -def get_year_start(dt, as_str=False): +def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: + """Returns the start date of the year for the given date (`dt`).""" dt = getdate(dt) date = datetime.date(dt.year, 1, 1) return date.strftime(DATE_FORMAT) if as_str else date @@ -535,7 +538,7 @@ def get_quarter_ending(date): def get_year_ending(date) -> datetime.date: - """returns year ending of the given date""" + """Returns year ending of the given date""" date = getdate(date) next_year_start = datetime.date(date.year + 1, 1, 1) return add_to_date(next_year_start, days=-1) From d578c98b50e08ff45a21c36ce316b2de4814930c Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:07:53 +0530 Subject: [PATCH 142/237] doc: get_datetime_str, get_date_str, get_time_str --- frappe/utils/data.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index fa4fb41af3..2287894cd7 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -565,19 +565,22 @@ def get_time( raise e -def get_datetime_str(datetime_obj) -> str: +def get_datetime_str(datetime_obj: DateTimeLikeObject) -> str: + """Returns the given datetime like object (datetime.date, datetime.datetime, string) as a string in `yyyy-mm-dd hh:mm:ss` format.""" if isinstance(datetime_obj, str): datetime_obj = get_datetime(datetime_obj) return datetime_obj.strftime(DATETIME_FORMAT) -def get_date_str(date_obj) -> str: +def get_date_str(date_obj: DateTimeLikeObject) -> str: + """Returns the given datetime like object (datetime.date, datetime.datetime, string) as a string in `yyyy-mm-dd` format.""" if isinstance(date_obj, str): date_obj = get_datetime(date_obj) return date_obj.strftime(DATE_FORMAT) -def get_time_str(timedelta_obj) -> str: +def get_time_str(timedelta_obj: datetime.timedelta | str) -> str: + """Returns the given timedelta object as a string in `hh:mm:ss` format.""" if isinstance(timedelta_obj, str): timedelta_obj = to_timedelta(timedelta_obj) From 6ffde4647ba93fabd671ca350353791924221254 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:16:47 +0530 Subject: [PATCH 143/237] refactor: support `str` in get_weekday --- frappe/utils/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 2287894cd7..20b52fa565 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -780,6 +780,10 @@ def get_weekdays(): def get_weekday(datetime: datetime.datetime | None = None) -> str: if not datetime: datetime = now_datetime() + + if isinstance(datetime, str): + datetime = get_datetime(datetime) + weekdays = get_weekdays() return weekdays[datetime.weekday()] From 73b73391630fa1b62cc1cbdf948a2694ec533ac4 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:17:28 +0530 Subject: [PATCH 144/237] doc: get_weekdays, get_weekday --- frappe/utils/data.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 20b52fa565..7217e738ab 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -773,11 +773,19 @@ def validate_duration_format(duration): ) -def get_weekdays(): +def get_weekdays() -> list[str]: + """Returns a list of weekday names. + + Return value: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] + """ return ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] -def get_weekday(datetime: datetime.datetime | None = None) -> str: +def get_weekday(datetime: DateTimeLikeObject | None = None) -> str: + """Returns the weekday name (e.g. 'Sunday') for the given datetime like object (datetime.date, datetime.datetime, string). + + If `datetime` argument is not provided, the current weekday name is returned. + """ if not datetime: datetime = now_datetime() From b42bcd53250ff24d9e15e1ca2c248bcca7e99c95 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:40:33 +0530 Subject: [PATCH 145/237] docs: get_first_day_of_week --- frappe/utils/data.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 7217e738ab..c0468a2324 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -475,7 +475,11 @@ def get_quarter_start(dt, as_str: bool = False) -> str | datetime.date: return first_date_of_quarter.strftime(DATE_FORMAT) if as_str else first_date_of_quarter -def get_first_day_of_week(dt, as_str=False): +def get_first_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date | str: + """Returns the first day of the week (as per System Settings or Sunday by default) for the given datetime like object (`dt`). + + If `as_str` is True, the first day of the week is returned as a string in `yyyy-mm-dd` format. + """ dt = getdate(dt) date = dt - datetime.timedelta(days=get_week_start_offset_days(dt)) return date.strftime(DATE_FORMAT) if as_str else date @@ -503,7 +507,8 @@ def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: return date.strftime(DATE_FORMAT) if as_str else date -def get_last_day_of_week(dt): +def get_last_day_of_week(dt: DateTimeLikeObject) -> datetime.date: + """Returns the last day of the week (first day as per System Settings or Sunday by default) for the given datetime like object (`dt`).""" dt = get_first_day_of_week(dt) return dt + datetime.timedelta(days=6) From 3de84281a000f1722286903ce44c3df73e2bdfdf Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:41:47 +0530 Subject: [PATCH 146/237] refactor: get_last_day_of_week supports `as_str` + docs --- frappe/utils/data.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index c0468a2324..a25cefa6f6 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -507,10 +507,14 @@ def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: return date.strftime(DATE_FORMAT) if as_str else date -def get_last_day_of_week(dt: DateTimeLikeObject) -> datetime.date: - """Returns the last day of the week (first day as per System Settings or Sunday by default) for the given datetime like object (`dt`).""" +def get_last_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date: + """Returns the last day of the week (first day as per System Settings or Sunday by default) for the given datetime like object (`dt`). + + If `as_str` is True, the last day of the week is returned as a string in `yyyy-mm-dd` format. + """ dt = get_first_day_of_week(dt) - return dt + datetime.timedelta(days=6) + date = dt + datetime.timedelta(days=6) + return date.strftime(DATE_FORMAT) if as_str else date def get_last_day(dt): From 5e619db45a4a3c9f17626cce690a089ee11770bd Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:42:39 +0530 Subject: [PATCH 147/237] fix: words --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index a25cefa6f6..eb62ffeccb 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -508,7 +508,7 @@ def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: def get_last_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date: - """Returns the last day of the week (first day as per System Settings or Sunday by default) for the given datetime like object (`dt`). + """Returns the last day of the week (first day is taken from System Settings or Sunday by default) for the given datetime like object (`dt`). If `as_str` is True, the last day of the week is returned as a string in `yyyy-mm-dd` format. """ From cdbe3555f655b4e565647c95d661f52557d4a0d3 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:43:30 +0530 Subject: [PATCH 148/237] fix: type hint --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index eb62ffeccb..40e8685484 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -507,7 +507,7 @@ def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: return date.strftime(DATE_FORMAT) if as_str else date -def get_last_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date: +def get_last_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date | str: """Returns the last day of the week (first day is taken from System Settings or Sunday by default) for the given datetime like object (`dt`). If `as_str` is True, the last day of the week is returned as a string in `yyyy-mm-dd` format. From d0d78f65e971d49999e4cdb41369333c425f6f73 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:48:45 +0530 Subject: [PATCH 149/237] docs: typing overloads for get_first_day_of_week/last_day_of_week --- frappe/utils/data.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 40e8685484..261996a5ee 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -475,6 +475,16 @@ def get_quarter_start(dt, as_str: bool = False) -> str | datetime.date: return first_date_of_quarter.strftime(DATE_FORMAT) if as_str else first_date_of_quarter +@typing.overload +def get_first_day_of_week(dt: DateTimeLikeObject, as_str: Literal[False] = False) -> datetime.date: + ... + + +@typing.overload +def get_first_day_of_week(dt: DateTimeLikeObject, as_str: Literal[True] = False) -> str: + ... + + def get_first_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date | str: """Returns the first day of the week (as per System Settings or Sunday by default) for the given datetime like object (`dt`). @@ -507,6 +517,16 @@ def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: return date.strftime(DATE_FORMAT) if as_str else date +@typing.overload +def get_last_day_of_week(dt: DateTimeLikeObject, as_str: Literal[False] = False) -> datetime.date: + ... + + +@typing.overload +def get_last_day_of_week(dt: DateTimeLikeObject, as_str: Literal[True] = False) -> str: + ... + + def get_last_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date | str: """Returns the last day of the week (first day is taken from System Settings or Sunday by default) for the given datetime like object (`dt`). From 3899c14b67fa94bd32c34f39c26c9a0fb259a5a1 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 14:54:59 +0530 Subject: [PATCH 150/237] refactor: get_quarter_start returns current when dt is None --- frappe/utils/data.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 261996a5ee..1ac6822a96 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -459,16 +459,24 @@ def get_first_day( @typing.overload -def get_quarter_start(dt, as_str: Literal[False] = False) -> datetime.date: +def get_quarter_start( + dt: DateTimeLikeObject | None = None, as_str: Literal[False] = False +) -> datetime.date: ... @typing.overload -def get_quarter_start(dt, as_str: Literal[True] = False) -> str: +def get_quarter_start(dt: DateTimeLikeObject | None = None, as_str: Literal[True] = False) -> str: ... -def get_quarter_start(dt, as_str: bool = False) -> str | datetime.date: +def get_quarter_start( + dt: DateTimeLikeObject | None = None, as_str: bool = False +) -> str | datetime.date: + """Returns the start date of the quarter for the given datetime like object (`dt`). + + If `dt` is None, the current quarter start date is returned. + """ date = getdate(dt) quarter = (date.month - 1) // 3 + 1 first_date_of_quarter = datetime.date(date.year, ((quarter - 1) * 3) + 1, 1) From 20f99819d5d36984d453e1cb5d907343c3094723 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 15:21:10 +0530 Subject: [PATCH 151/237] docs: typing overloads for get_year_start --- frappe/utils/data.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 1ac6822a96..63d6b511a9 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -518,6 +518,16 @@ def get_normalized_weekday_index(dt): return (dt.weekday() + 1) % 7 +@typing.overload +def get_year_start(dt: DateTimeLikeObject, as_str: Literal[False] = False) -> datetime.date: + ... + + +@typing.overload +def get_year_start(dt: DateTimeLikeObject, as_str: Literal[True] = False) -> str: + ... + + def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: """Returns the start date of the year for the given date (`dt`).""" dt = getdate(dt) From 10d34da769592cc39d14c9ee447a8c7211eb16e7 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 15:28:54 +0530 Subject: [PATCH 152/237] docs: get_quarter_ending --- frappe/utils/data.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 63d6b511a9..dd676e5812 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -569,7 +569,26 @@ def is_last_day_of_the_month(dt): return getdate(dt) == getdate(last_day_of_the_month) -def get_quarter_ending(date): +@typing.overload +def get_quarter_ending( + dt: DateTimeLikeObject | None = None, as_str: Literal[False] = False +) -> datetime.date: + ... + + +@typing.overload +def get_quarter_ending(dt: DateTimeLikeObject | None = None, as_str: Literal[True] = False) -> str: + ... + + +def get_quarter_ending( + date: DateTimeLikeObject | None = None, as_str=False +) -> str | datetime.date: + """Returns the end date of the quarter for the given datetime like object (`date`). + + If `date` is None, the current quarter end date is returned. + If `as_str` is True, the end date of the quarter is returned as a string in `yyyy-mm-dd` format. + """ date = getdate(date) # find the earliest quarter ending date that is after @@ -581,7 +600,7 @@ def get_quarter_ending(date): date = quarter_end_date break - return date + return date.strftime(DATE_FORMAT) if as_str else date def get_year_ending(date) -> datetime.date: From 45c491228b399b949f0c77cf4a58f429b26ac4c0 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 15:34:07 +0530 Subject: [PATCH 153/237] docs: get_year_ending --- frappe/utils/data.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index dd676e5812..b2eb74da35 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -603,11 +603,28 @@ def get_quarter_ending( return date.strftime(DATE_FORMAT) if as_str else date -def get_year_ending(date) -> datetime.date: - """Returns year ending of the given date""" +@typing.overload +def get_year_ending( + dt: DateTimeLikeObject | None = None, as_str: Literal[False] = False +) -> datetime.date: + ... + + +@typing.overload +def get_year_ending(dt: DateTimeLikeObject | None = None, as_str: Literal[True] = False) -> str: + ... + + +def get_year_ending(date: DateTimeLikeObject | None = None, as_str=False) -> datetime.date | str: + """Returns the end date of the year for the given datetime like object (`date`). + + If `date` is None, the current year end date is returned. + If `as_str` is True, the end date of the year is returned as a string in `yyyy-mm-dd` format. + """ date = getdate(date) next_year_start = datetime.date(date.year + 1, 1, 1) - return add_to_date(next_year_start, days=-1) + year_ending = add_to_date(next_year_start, days=-1) + return year_ending.strftime(DATE_FORMAT) if as_str else year_ending def get_time( From 0b95e8fcc2f80fa68a2d318db4abfb2b58b9c0d7 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 15:35:55 +0530 Subject: [PATCH 154/237] chore: get_year_ending to safe_exec --- frappe/utils/safe_exec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index d8b30b871d..a3f313cb77 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -576,6 +576,7 @@ VALID_UTILS = ( "get_quarter_ending", "get_first_day_of_week", "get_year_start", + "get_year_ending", "get_last_day_of_week", "get_last_day", "get_time", From bd46a75aff1581081b8b02f7be3e8fa8792cf99b Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 15:49:34 +0530 Subject: [PATCH 155/237] docs: get_timespan_date_range --- frappe/utils/data.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index b2eb74da35..3b49f3e395 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -25,6 +25,25 @@ from frappe.desk.utils import slug DateTimeLikeObject = Union[str, datetime.date, datetime.datetime] NumericType = Union[int, float] +TimespanOptions = Literal[ + "last week", + "last month", + "last quarter", + "last 6 months", + "last year", + "yesterday", + "today", + "tomorrow", + "this week", + "this month", + "this quarter", + "this year", + "next week", + "next month", + "next quarter", + "next 6 months", + "next year", +] if typing.TYPE_CHECKING: @@ -879,7 +898,10 @@ def get_weekday(datetime: DateTimeLikeObject | None = None) -> str: return weekdays[datetime.weekday()] -def get_timespan_date_range(timespan: str) -> tuple[datetime.datetime, datetime.datetime] | None: +def get_timespan_date_range( + timespan: TimespanOptions, +) -> tuple[datetime.datetime, datetime.datetime] | None: + """Returns the date range (start_date, end_date) tuple for the given timespan.""" today = getdate() match timespan: From 75d81da634e75395dd60b4e6e39e0531e506898a Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 16:35:29 +0530 Subject: [PATCH 156/237] fix: unicode handles more than just `str` --- frappe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 67cae7556d..e8871542bd 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -144,7 +144,7 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str: return translated_string or non_translated_string -def as_unicode(text: str, encoding: str = "utf-8") -> str: +def as_unicode(text, encoding: str = "utf-8") -> str: """Convert to unicode if required""" if isinstance(text, str): return text From f566b6cf4c59b4315d65d22618cfd189b5842237 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 16:36:25 +0530 Subject: [PATCH 157/237] docs: cstr --- frappe/utils/data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 3b49f3e395..fed0c6efd0 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1188,7 +1188,8 @@ def ceil(s): return num -def cstr(s, encoding="utf-8"): +def cstr(s, encoding="utf-8") -> str: + """Converts the given argument to string.""" return frappe.as_unicode(s, encoding) From 751e21bf7375ebb5846615433f351260c234613a Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 16:54:57 +0530 Subject: [PATCH 158/237] docs: is_image, is_html, comma_or, comma_and, new_line_sep --- frappe/utils/data.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index fed0c6efd0..8973a8ff55 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1574,12 +1574,14 @@ def in_words(integer: int, in_million=True) -> str: def is_html(text: str) -> bool: + """Returns True if the given `text` contains any HTML tags.""" if not isinstance(text, str): return False return HTML_TAG_PATTERN.search(text) def is_image(filepath: str) -> bool: + """Returns True if the given `filepath` points to an image file.""" from mimetypes import guess_type # filepath can be https://example.com/bed.jpg?v=129 @@ -1696,15 +1698,27 @@ def pretty_date(iso_datetime: datetime.datetime | str) -> str: return format_timedelta(iso_datetime - now_dt, add_direction=True, locale=locale) -def comma_or(some_list, add_quotes=True): +def comma_or(some_list: list | tuple, add_quotes=True) -> str: + """Returns the given list or tuple as a comma separated string with the last item joined by 'or'. + e.g. ['a', 'b', 'c'] -> 'a, b or c' + + If `add_quotes` is True, each item in the list will be wrapped in single quotes. + e.g. ['a', 'b', 'c'] -> "'a', 'b' or 'c'" + """ return comma_sep(some_list, frappe._("{0} or {1}"), add_quotes) -def comma_and(some_list, add_quotes=True): +def comma_and(some_list: list | tuple, add_quotes=True) -> str: + """Returns the given list or tuple as a comma separated string with the last item joined by 'and'. + e.g. ['a', 'b', 'c'] -> 'a, b and c' + + If `add_quotes` is True, each item in the list will be wrapped in single quotes. + e.g. ['a', 'b', 'c'] -> "'a', 'b' and 'c'" + """ return comma_sep(some_list, frappe._("{0} and {1}"), add_quotes) -def comma_sep(some_list, pattern, add_quotes=True): +def comma_sep(some_list: list | tuple, pattern: str, add_quotes=True) -> str: if isinstance(some_list, (list, tuple)): # list(some_list) is done to preserve the existing list some_list = [str(s) for s in list(some_list)] @@ -1719,7 +1733,11 @@ def comma_sep(some_list, pattern, add_quotes=True): return some_list -def new_line_sep(some_list): +def new_line_sep(some_list: list | tuple) -> str: + """Returns the given list or tuple as a new line separated string. + + e.g. ['', 'Paid', 'Unpaid'] -> '\n Paid\n Unpaid' + """ if isinstance(some_list, (list, tuple)): # list(some_list) is done to preserve the existing list some_list = [str(s) for s in list(some_list)] From 6ff2023a206d51111e55fec2fad84abae26eadd5 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 17:02:42 +0530 Subject: [PATCH 159/237] docs: comma_sep --- frappe/utils/data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 8973a8ff55..420cd69e02 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1719,6 +1719,12 @@ def comma_and(some_list: list | tuple, add_quotes=True) -> str: def comma_sep(some_list: list | tuple, pattern: str, add_quotes=True) -> str: + """Returns the given list or tuple as a comma separated string, with the last item joined by the given string format pattern. + + If `add_quotes` is True, each item in the list will be wrapped in single quotes. + + e.g. if `some_list` is ['a', 'b', 'c'] and `pattern` is '{0} or {1}', the output will be 'a, b or c' + """ if isinstance(some_list, (list, tuple)): # list(some_list) is done to preserve the existing list some_list = [str(s) for s in list(some_list)] From b174224f21a3f284367a8ffa17b45c46a5c1db38 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 17:05:54 +0530 Subject: [PATCH 160/237] docs: url_contains_port --- frappe/utils/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 420cd69e02..6ce1f17362 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1828,6 +1828,10 @@ def get_host_name_from_request() -> str: def url_contains_port(url: str) -> bool: + """Returns True if the given url contains a port number. + + e.g. 'http://localhost:8000' -> True, 'http://localhost' -> False + """ parts = url.split(":") return len(parts) > 2 From e52b9049fd4cb0494ba9a5dfd0d0cf31a345c21e Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 17:17:30 +0530 Subject: [PATCH 161/237] docs: get_absolute_url --- frappe/utils/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 6ce1f17362..cf9929c89f 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1877,6 +1877,10 @@ def get_link_to_report( def get_absolute_url(doctype: str, name: str) -> str: + """Returns the absolute route for the form view of the given document in the desk. + + e.g. when doctype="Sales Invoice" and name="INV-00001", returns '/app/sales-invoice/INV-00001' + """ return f"/app/{quoted(slug(doctype))}/{quoted(name)}" From 93a5a60003ef2541012c501feb73a1f56b8d2e13 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 17:22:15 +0530 Subject: [PATCH 162/237] doc: get_url_to_list --- frappe/utils/data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index cf9929c89f..f693d0e095 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1889,6 +1889,11 @@ def get_url_to_form(doctype: str, name: str) -> str: def get_url_to_list(doctype: str) -> str: + """Returns the absolute URL for the list view of the given document in the desk. + + e.g. when doctype="Sales Invoice" and your site URL is "https://frappe.io", + returns 'https://frappe.io/app/sales-invoice' + """ return get_url(uri=f"/app/{quoted(slug(doctype))}") From db10004fe033a9be1817ded23438462ecdd58ed9 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 17:26:01 +0530 Subject: [PATCH 163/237] docs: markdown, to_markdown, md_to_html --- frappe/utils/data.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index f693d0e095..5fa31b8498 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -2181,6 +2181,7 @@ def get_string_between(start: str, string: str, end: str) -> str: def to_markdown(html: str) -> str: + """Converts the given HTML to markdown and returns it.""" from html.parser import HTMLParser from frappe.core.utils import html2text @@ -2192,6 +2193,7 @@ def to_markdown(html: str) -> str: def md_to_html(markdown_text: str) -> Optional["UnicodeWithAttrs"]: + """Converts the given markdown text to HTML and returns it.""" from markdown2 import MarkdownError from markdown2 import markdown as _markdown @@ -2210,7 +2212,8 @@ def md_to_html(markdown_text: str) -> Optional["UnicodeWithAttrs"]: pass -def markdown(markdown_text): +def markdown(markdown_text: str) -> str: + """Converts the given markdown text to HTML and returns it.""" return md_to_html(markdown_text) From 837b6ff995759b6d5a11df9bdfefb1fac9b0532e Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Sun, 17 Dec 2023 17:29:32 +0530 Subject: [PATCH 164/237] docs: generate_hash --- frappe/utils/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 5fa31b8498..8b56f9509e 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -2223,6 +2223,10 @@ def is_subset(list_a: list, list_b: list) -> bool: def generate_hash(*args, **kwargs) -> str: + """Generates a random hash using best available randomness source and returns it. + + You can optionally provide the `length` of the hash to be generated. Default is 56. + """ return frappe.generate_hash(*args, **kwargs) From f110b6eea3dffc6da45152b08c98c5a2659e64b0 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 18 Dec 2023 10:56:51 +0530 Subject: [PATCH 165/237] refactor(data_import): handle RQ timeouts better (#23811) * refactor(data_import): handle RQ timeouts better Signed-off-by: Akhil Narang * refactor(data_import): display count of documents even for timed out jobs Also handle "0" cases better - should be plural there Signed-off-by: Akhil Narang --------- Signed-off-by: Akhil Narang --- .../core/doctype/data_import/data_import.js | 59 ++- .../core/doctype/data_import/data_import.json | 394 +++++++++--------- .../core/doctype/data_import/data_import.py | 12 +- .../doctype/data_import/data_import_list.js | 3 +- 4 files changed, 236 insertions(+), 232 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 7db3aa9629..b3fa136eb4 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -136,47 +136,40 @@ frappe.ui.form.on("Data Import", { let total_records = cint(r.message.total_records); if (!total_records) return; + let action, message; + if (frm.doc.import_type === "Insert New Records") { + action = "imported"; + } else { + action = "updated"; + } - let message; if (failed_records === 0) { - let message_args = [successful_records]; - if (frm.doc.import_type === "Insert New Records") { - message = - successful_records > 1 - ? __("Successfully imported {0} records.", message_args) - : __("Successfully imported {0} record.", message_args); + let message_args = [action, successful_records]; + if (successful_records === 1) { + message = __("Successfully {0} 1 record.", message_args); } else { - message = - successful_records > 1 - ? __("Successfully updated {0} records.", message_args) - : __("Successfully updated {0} record.", message_args); + message = __("Successfully {0} {1} records.", message_args); } } else { - let message_args = [successful_records, total_records]; - if (frm.doc.import_type === "Insert New Records") { - message = - successful_records > 1 - ? __( - "Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ) - : __( - "Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ); + let message_args = [action, successful_records, total_records]; + if (successful_records === 1) { + message = __( + "Successfully {0} {1} record out of {2}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); } else { - message = - successful_records > 1 - ? __( - "Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ) - : __( - "Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ); + message = __( + "Successfully {0} {1} records out of {2}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); } } + + // If the job timed out, display an extra hint + if (r.message.status === "Timed Out") { + message += "
" + __("Import timed out, please re-try."); + } + frm.dashboard.set_headline(message); }, }); diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index faa9a33bf1..97716219a2 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -1,198 +1,198 @@ { - "actions": [], - "autoname": "format:{reference_doctype} Import on {creation}", - "beta": 1, - "creation": "2019-08-04 14:16:08.318714", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "import_type", - "download_template", - "import_file", - "payload_count", - "html_5", - "google_sheets_url", - "refresh_google_sheet", - "column_break_5", - "status", - "submit_after_import", - "mute_emails", - "template_options", - "import_warnings_section", - "template_warnings", - "import_warnings", - "section_import_preview", - "import_preview", - "import_log_section", - "show_failed_logs", - "import_log_preview" - ], - "fields": [ - { - "fieldname": "reference_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1, - "set_only_once": 1 - }, - { - "fieldname": "import_type", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Import Type", - "options": "\nInsert New Records\nUpdate Existing Records", - "reqd": 1, - "set_only_once": 1 - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "import_file", - "fieldtype": "Attach", - "in_list_view": 1, - "label": "Import File", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "fieldname": "import_preview", - "fieldtype": "HTML", - "label": "Import Preview" - }, - { - "fieldname": "section_import_preview", - "fieldtype": "Section Break", - "label": "Preview" - }, - { - "fieldname": "column_break_5", - "fieldtype": "Column Break" - }, - { - "fieldname": "template_options", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Options", - "options": "JSON", - "read_only": 1 - }, - { - "fieldname": "import_log_section", - "fieldtype": "Section Break", - "label": "Import Log" - }, - { - "fieldname": "import_log_preview", - "fieldtype": "HTML", - "label": "Import Log Preview" - }, - { - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 1, - "label": "Status", - "no_copy": 1, - "options": "Pending\nSuccess\nPartial Success\nError", - "read_only": 1 - }, - { - "fieldname": "template_warnings", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Warnings", - "options": "JSON" - }, - { - "default": "0", - "fieldname": "submit_after_import", - "fieldtype": "Check", - "label": "Submit After Import", - "set_only_once": 1 - }, - { - "fieldname": "import_warnings_section", - "fieldtype": "Section Break", - "label": "Import File Errors and Warnings" - }, - { - "fieldname": "import_warnings", - "fieldtype": "HTML", - "label": "Import Warnings" - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "download_template", - "fieldtype": "Button", - "label": "Download Template" - }, - { - "default": "1", - "fieldname": "mute_emails", - "fieldtype": "Check", - "label": "Don't Send Emails", - "set_only_once": 1 - }, - { - "default": "0", - "fieldname": "show_failed_logs", - "fieldtype": "Check", - "label": "Show Failed Logs" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file", - "fieldname": "html_5", - "fieldtype": "HTML", - "options": "
Or
" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file\n", - "description": "Must be a publicly accessible Google Sheets URL", - "fieldname": "google_sheets_url", - "fieldtype": "Data", - "label": "Import from Google Sheets", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", - "fieldname": "refresh_google_sheet", - "fieldtype": "Button", - "label": "Refresh Google Sheet" - }, - { - "fieldname": "payload_count", - "fieldtype": "Int", - "hidden": 1, - "label": "Payload Count", - "read_only": 1 - } - ], - "hide_toolbar": 1, - "links": [], - "modified": "2022-02-14 10:08:37.624914", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import", - "naming_rule": "Expression", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "states": [], - "track_changes": 1 -} + "actions": [], + "autoname": "format:{reference_doctype} Import on {creation}", + "beta": 1, + "creation": "2019-08-04 14:16:08.318714", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "import_type", + "download_template", + "import_file", + "payload_count", + "html_5", + "google_sheets_url", + "refresh_google_sheet", + "column_break_5", + "status", + "submit_after_import", + "mute_emails", + "template_options", + "import_warnings_section", + "template_warnings", + "import_warnings", + "section_import_preview", + "import_preview", + "import_log_section", + "show_failed_logs", + "import_log_preview" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "import_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Import Type", + "options": "\nInsert New Records\nUpdate Existing Records", + "reqd": 1, + "set_only_once": 1 + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "import_file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Import File", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "fieldname": "import_preview", + "fieldtype": "HTML", + "label": "Import Preview" + }, + { + "fieldname": "section_import_preview", + "fieldtype": "Section Break", + "label": "Preview" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "template_options", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Options", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "import_log_section", + "fieldtype": "Section Break", + "label": "Import Log" + }, + { + "fieldname": "import_log_preview", + "fieldtype": "HTML", + "label": "Import Log Preview" + }, + { + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "label": "Status", + "no_copy": 1, + "options": "Pending\nSuccess\nPartial Success\nError\nTimed Out", + "read_only": 1 + }, + { + "fieldname": "template_warnings", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Warnings", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "label": "Submit After Import", + "set_only_once": 1 + }, + { + "fieldname": "import_warnings_section", + "fieldtype": "Section Break", + "label": "Import File Errors and Warnings" + }, + { + "fieldname": "import_warnings", + "fieldtype": "HTML", + "label": "Import Warnings" + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" + }, + { + "default": "1", + "fieldname": "mute_emails", + "fieldtype": "Check", + "label": "Don't Send Emails", + "set_only_once": 1 + }, + { + "default": "0", + "fieldname": "show_failed_logs", + "fieldtype": "Check", + "label": "Show Failed Logs" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file", + "fieldname": "html_5", + "fieldtype": "HTML", + "options": "
Or
" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file\n", + "description": "Must be a publicly accessible Google Sheets URL", + "fieldname": "google_sheets_url", + "fieldtype": "Data", + "label": "Import from Google Sheets", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", + "fieldname": "refresh_google_sheet", + "fieldtype": "Button", + "label": "Refresh Google Sheet" + }, + { + "fieldname": "payload_count", + "fieldtype": "Int", + "hidden": 1, + "label": "Payload Count", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "links": [], + "modified": "2023-12-15 12:45:49.452834", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index bd6c6efe4f..f3dca2d5af 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -3,6 +3,8 @@ import os +from rq.timeouts import JobTimeoutException + import frappe from frappe import _ from frappe.core.doctype.data_import.exporter import Exporter @@ -32,11 +34,13 @@ class DataImport(Document): payload_count: DF.Int reference_doctype: DF.Link show_failed_logs: DF.Check - status: DF.Literal["Pending", "Success", "Partial Success", "Error"] + status: DF.Literal["Pending", "Success", "Partial Success", "Error", "Timed Out"] submit_after_import: DF.Check template_options: DF.Code | None template_warnings: DF.Code | None + # end: auto-generated types + def validate(self): doc_before_save = self.get_doc_before_save() if ( @@ -136,6 +140,9 @@ def start_import(data_import): try: i = Importer(data_import.reference_doctype, data_import=data_import) i.import_data() + except JobTimeoutException: + frappe.db.rollback() + data_import.db_set("status", "Timed Out") except Exception: frappe.db.rollback() data_import.db_set("status", "Error") @@ -190,6 +197,9 @@ def download_import_log(data_import_name): def get_import_status(data_import_name): import_status = {} + data_import = frappe.get_doc("Data Import", data_import_name) + import_status["status"] = data_import.status + logs = frappe.get_all( "Data Import Log", fields=["count(*) as count", "success"], diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index c054655e62..a16478cdb1 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -20,13 +20,14 @@ frappe.listview_settings["Data Import"] = { Success: "green", "In Progress": "orange", Error: "red", + "Timed Out": "orange", }; let status = doc.status; if (imports_in_progress.includes(doc.name)) { status = "In Progress"; } - if (status == "Pending") { + if (status === "Pending") { status = "Not Started"; } From 806708650cfdc952d15d6b559422d8781a31b8dc Mon Sep 17 00:00:00 2001 From: Gursheen Anand Date: Mon, 18 Dec 2023 12:15:51 +0530 Subject: [PATCH 166/237] chore: linter issue --- frappe/desk/query_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index bc12f52955..161e45c75e 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -397,7 +397,7 @@ def build_xlsx_data( if not filter_value: continue filter_value = ( - ", ".join(map(lambda x: cstr(x), filter_value)) + ", ".join([cstr(x) for x in filter_value]) if isinstance(filter_value, list) else cstr(filter_value) ) From 85a661f273558129dd710428163409fc988c1a28 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 18 Dec 2023 12:29:20 +0530 Subject: [PATCH 167/237] test: flakey currency UI test (#23834) --- cypress/integration/control_currency.js | 2 ++ cypress/integration/control_float.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cypress/integration/control_currency.js b/cypress/integration/control_currency.js index 1fb912d9ff..9db5dee2a3 100644 --- a/cypress/integration/control_currency.js +++ b/cypress/integration/control_currency.js @@ -9,6 +9,7 @@ context("Control Currency", () => { function get_dialog_with_currency(df_options = {}) { return cy.dialog({ title: "Currency Check", + animate: false, fields: [ { fieldname: fieldname, @@ -76,6 +77,7 @@ context("Control Currency", () => { }); get_dialog_with_currency(test_case.df_options).as("dialog"); + cy.wait(300); cy.get_field(fieldname, "Currency").clear(); cy.wait(300); cy.fill_field(fieldname, test_case.input, "Currency").blur(); diff --git a/cypress/integration/control_float.js b/cypress/integration/control_float.js index 08b71eb870..25936066cd 100644 --- a/cypress/integration/control_float.js +++ b/cypress/integration/control_float.js @@ -7,6 +7,7 @@ context("Control Float", () => { function get_dialog_with_float() { return cy.dialog({ title: "Float Check", + animate: false, fields: [ { fieldname: "float_number", @@ -19,6 +20,7 @@ context("Control Float", () => { it("check value changes", () => { get_dialog_with_float().as("dialog"); + cy.wait(300); let data = get_data(); data.forEach((x) => { From 47626e96fb86e92cb1fd50dc870a4e4537962fff Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 12:44:07 +0530 Subject: [PATCH 168/237] fix: type hint --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 8b56f9509e..d8a8ae4b01 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -2212,7 +2212,7 @@ def md_to_html(markdown_text: str) -> Optional["UnicodeWithAttrs"]: pass -def markdown(markdown_text: str) -> str: +def markdown(markdown_text: str) -> Optional["UnicodeWithAttrs"]: """Converts the given markdown text to HTML and returns it.""" return md_to_html(markdown_text) From aefe634ff7da58f39c26862c68c21f6a6c9afb08 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 18:20:52 +0530 Subject: [PATCH 169/237] docs: password_strength.py * also add type hints --- frappe/utils/password_strength.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/frappe/utils/password_strength.py b/frappe/utils/password_strength.py index eada58c638..4f24bc61df 100644 --- a/frappe/utils/password_strength.py +++ b/frappe/utils/password_strength.py @@ -1,14 +1,18 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from zxcvbn import zxcvbn +from collections.abc import Iterable + +from zxcvbn import _Result, zxcvbn +from zxcvbn.feedback import _Feedback as PasswordStrengthFeedback +from zxcvbn.matching import _Match from zxcvbn.scoring import ALL_UPPER, START_UPPER import frappe from frappe import _ -def test_password_strength(password, user_inputs=None): +def test_password_strength(password: str, user_inputs: Iterable[object] = None) -> _Result: """Wrapper around zxcvbn.password_strength""" if len(password) > 128: # zxcvbn takes forever when checking long, random passwords. @@ -27,8 +31,9 @@ def test_password_strength(password, user_inputs=None): # see license for feedback code at https://github.com/sans-serif/python-zxcvbn/blob/master/LICENSE.txt # ------------------------------------------- + # Default feedback value -default_feedback = { +default_feedback: PasswordStrengthFeedback = { "warning": "", "suggestions": [ _("Use a few words, avoid common phrases."), @@ -37,10 +42,8 @@ default_feedback = { } -def get_feedback(score, sequence): - """ - Returns the feedback dictionary consisting of ("warning","suggestions") for the given sequences. - """ +def get_feedback(score: int, sequence: list) -> PasswordStrengthFeedback: + """Return the feedback dictionary consisting of ("warning","suggestions") for the given sequences.""" global default_feedback minimum_password_score = int( frappe.db.get_single_value("System Settings", "minimum_password_score") or 2 @@ -69,10 +72,8 @@ def get_feedback(score, sequence): return feedback -def get_match_feedback(match, is_sole_match): - """ - Returns feedback as a dictionary for a certain match - """ +def get_match_feedback(match: _Match, is_sole_match: bool) -> PasswordStrengthFeedback: + """Return feedback as a dictionary for a certain match.""" def fun_bruteforce(): # Define a number of functions that are used in a look up dictionary @@ -142,10 +143,8 @@ def get_match_feedback(match, is_sole_match): return pattern_fn() -def get_dictionary_match_feedback(match, is_sole_match): - """ - Returns feedback for a match that is found in a dictionary - """ +def get_dictionary_match_feedback(match: _Match, is_sole_match: bool) -> PasswordStrengthFeedback: + """Return feedback for a match that is found in a dictionary.""" warning = "" suggestions = [] From 8d2137c265ed7fb745523327c22d33c33019b87f Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 18:27:39 +0530 Subject: [PATCH 170/237] docs: consistent doc strings --- frappe/__init__.py | 46 ++-- frappe/auth.py | 2 +- .../doctype/auto_repeat/auto_repeat.py | 6 +- frappe/build.py | 6 +- frappe/client.py | 15 +- frappe/contacts/doctype/address/address.py | 4 +- frappe/contacts/doctype/contact/contact.py | 2 +- frappe/core/api/file.py | 3 +- .../doctype/communication/communication.py | 14 +- frappe/core/doctype/communication/mixins.py | 4 +- frappe/core/doctype/data_import/importer.py | 4 +- frappe/core/doctype/docfield/docfield.py | 7 +- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/core/doctype/file/file.py | 4 +- frappe/core/doctype/page/page.py | 2 +- frappe/core/doctype/report/report.py | 2 +- .../doctype/server_script/server_script.py | 26 +-- .../core/doctype/sms_settings/sms_settings.py | 2 +- frappe/core/doctype/user/user.py | 12 +- .../user_permission/user_permission.py | 2 +- frappe/core/notifications.py | 4 +- .../permission_manager/permission_manager.py | 10 +- frappe/core/utils.py | 10 +- frappe/database/database.py | 30 +-- frappe/database/mariadb/database.py | 8 +- frappe/database/operator_map.py | 24 +- frappe/database/postgres/database.py | 6 +- frappe/database/query.py | 2 +- frappe/defaults.py | 2 +- frappe/desk/calendar.py | 2 +- frappe/desk/desktop.py | 13 +- frappe/desk/doctype/tag/tag.py | 25 +- frappe/desk/doctype/todo/todo.py | 2 +- frappe/desk/form/linked_with.py | 8 +- frappe/desk/form/load.py | 2 +- frappe/desk/query_report.py | 15 +- .../auto_email_report/auto_email_report.py | 2 +- .../doctype/email_account/email_account.py | 2 +- .../doctype/email_template/email_template.py | 2 +- frappe/email/email_body.py | 4 +- frappe/email/receive.py | 4 +- frappe/frappeclient.py | 6 +- frappe/geo/utils.py | 4 +- frappe/installer.py | 6 +- .../google_calendar/google_calendar.py | 26 +-- .../google_contacts/google_contacts.py | 4 +- .../doctype/google_drive/google_drive.py | 4 +- .../oauth_provider_settings.py | 2 +- frappe/integrations/google_oauth.py | 6 +- frappe/integrations/utils.py | 10 +- frappe/migrate.py | 2 +- frappe/model/base_document.py | 27 +-- frappe/model/db_query.py | 7 +- frappe/model/document.py | 12 +- frappe/model/mapper.py | 11 +- frappe/model/meta.py | 16 +- frappe/model/naming.py | 2 +- frappe/model/rename_doc.py | 3 +- frappe/model/utils/__init__.py | 2 +- frappe/modules/import_file.py | 12 +- frappe/modules/utils.py | 13 +- .../v11_0/replicate_old_user_permissions.py | 2 +- frappe/patches/v13_0/queryreport_columns.py | 2 +- frappe/permissions.py | 30 ++- frappe/query_builder/custom.py | 6 +- frappe/query_builder/terms.py | 14 +- frappe/query_builder/utils.py | 7 +- frappe/rate_limiter.py | 2 +- frappe/search/full_text_search.py | 13 +- frappe/search/website_search.py | 14 +- frappe/sessions.py | 4 +- frappe/tests/test_commands.py | 18 +- frappe/translate.py | 16 +- frappe/twofactor.py | 2 +- frappe/utils/__init__.py | 57 ++--- frappe/utils/background_jobs.py | 6 +- frappe/utils/backups.py | 18 +- frappe/utils/change_log.py | 22 +- frappe/utils/data.py | 221 ++++++++---------- frappe/utils/file_lock.py | 2 +- frappe/utils/file_manager.py | 6 +- frappe/utils/identicon.py | 4 +- frappe/utils/image.py | 7 +- frappe/utils/jinja.py | 2 +- frappe/utils/logger.py | 5 +- frappe/utils/print_format.py | 9 +- frappe/utils/redis_wrapper.py | 14 +- frappe/utils/scheduler.py | 4 +- frappe/utils/user.py | 2 +- frappe/utils/weasyprint.py | 7 +- .../personal_data_download_request.py | 2 +- frappe/website/doctype/web_form/web_form.py | 2 +- frappe/website/doctype/web_page/web_page.py | 2 +- .../website_settings/google_indexing.py | 2 +- frappe/website/path_resolver.py | 2 +- frappe/website/router.py | 4 +- frappe/website/utils.py | 4 +- frappe/website/website_generator.py | 3 +- .../workflow_action/workflow_action.py | 4 +- frappe/www/list.py | 9 +- frappe/www/printview.py | 12 +- frappe/www/sitemap.py | 2 +- 102 files changed, 487 insertions(+), 594 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index e8871542bd..4d8ededeb9 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -109,7 +109,7 @@ class _dict(dict): def _(msg: str, lang: str | None = None, context: str | None = None) -> str: - """Returns translated string in current lang, if exists. + """Return translated string in current lang, if exists. Usage: _('Change') _('Change', context='Coins') @@ -145,7 +145,7 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str: def as_unicode(text, encoding: str = "utf-8") -> str: - """Convert to unicode if required""" + """Convert to unicode if required.""" if isinstance(text, str): return text elif text is None: @@ -327,7 +327,7 @@ def connect_replica() -> bool: def get_site_config(sites_path: str | None = None, site_path: str | None = None) -> dict[str, Any]: - """Returns `site_config.json` combined with `sites/common_site_config.json`. + """Return `site_config.json` combined with `sites/common_site_config.json`. `site_config` is a set of site wide settings like database name, password, email etc.""" config = _dict() @@ -373,7 +373,7 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None) def get_common_site_config(sites_path: str | None = None) -> dict[str, Any]: - """Returns common site config as dictionary. + """Return common site config as dictionary. This is useful for: - checking configuration which should only be allowed in common site config @@ -432,7 +432,7 @@ def setup_redis_cache_connection(): def get_traceback(with_context: bool = False) -> str: - """Returns error traceback.""" + """Return error traceback.""" from frappe.utils import get_traceback return get_traceback(with_context=with_context) @@ -638,7 +638,7 @@ def get_user(): def get_roles(username=None) -> list[str]: - """Returns roles of current user.""" + """Return roles of current user.""" if not local.session or not local.session.user: return ["Guest"] import frappe.permissions @@ -1003,8 +1003,8 @@ def has_permission( parent_doctype=None, ): """ - Returns True if the user has permission `ptype` for given `doctype` or `doc` - Raises `frappe.PermissionError` if user isn't permitted and `throw` is truthy + Return True if the user has permission `ptype` for given `doctype` or `doc` + Raise `frappe.PermissionError` if user isn't permitted and `throw` is truthy :param doctype: DocType for which permission is to be check. :param ptype: Permission type (`read`, `write`, `create`, `submit`, `cancel`, `amend`). Default: `read`. @@ -1084,7 +1084,7 @@ def has_website_permission(doc=None, ptype="read", user=None, verbose=False, doc def is_table(doctype: str) -> bool: - """Returns True if `istable` property (indicating child Table) is set for given DocType.""" + """Return True if `istable` property (indicating child Table) is set for given DocType.""" def get_tables(): return db.get_values("DocType", filters={"istable": 1}, order_by=None, pluck=True) @@ -1128,7 +1128,7 @@ def new_doc( as_dict: bool = False, **kwargs, ) -> "Document": - """Returns a new document of the given DocType with defaults set. + """Return a new document of the given DocType with defaults set. :param doctype: DocType of the new document. :param parent_doc: [optional] add to parent document. @@ -1174,7 +1174,7 @@ def _set_document_in_cache(key: str, doc: "Document") -> None: def can_cache_doc(args) -> str | None: """ Determine if document should be cached based on get_doc params. - Returns cache key if doc can be cached, None otherwise. + Return cache key if doc can be cached, None otherwise. """ if not args: @@ -1426,17 +1426,17 @@ def rename_doc( def get_module(modulename): - """Returns a module object for given Python module name using `importlib.import_module`.""" + """Return a module object for given Python module name using `importlib.import_module`.""" return importlib.import_module(modulename) def scrub(txt: str) -> str: - """Returns sluggified string. e.g. `Sales Order` becomes `sales_order`.""" + """Return sluggified string. e.g. `Sales Order` becomes `sales_order`.""" return cstr(txt).replace(" ", "_").replace("-", "_").lower() def unscrub(txt: str) -> str: - """Returns titlified string. e.g. `sales_order` becomes `Sales Order`.""" + """Return titlified string. e.g. `sales_order` becomes `Sales Order`.""" return txt.replace("_", " ").replace("-", " ").title() @@ -1536,7 +1536,7 @@ def get_installed_apps(*, _ensure_on_bench=False) -> list[str]: def get_doc_hooks(): - """Returns hooked methods for given doc. It will expand the dict tuple if required.""" + """Return hooked methods for given doc. Expand the dict tuple if required.""" if not hasattr(local, "doc_events_hooks"): hooks = get_hooks("doc_events", {}) out = {} @@ -1643,7 +1643,7 @@ def setup_module_map(): def get_file_items(path, raise_not_found=False, ignore_empty_lines=True): - """Returns items from text file as a list. Ignores empty lines.""" + """Return items from text file as a list. Ignore empty lines.""" import frappe.utils content = read_file(path, raise_not_found=raise_not_found) @@ -1996,7 +1996,7 @@ def get_all(doctype, *args, **kwargs): def get_value(*args, **kwargs): - """Returns a document property or list of properties. + """Return a document property or list of properties. Alias for `frappe.db.get_value` @@ -2011,7 +2011,7 @@ def get_value(*args, **kwargs): def as_json(obj: dict | list, indent=1, separators=None, ensure_ascii=True) -> str: - """Returns the JSON string representation of the given `obj`.""" + """Return the JSON string representation of the given `obj`.""" from frappe.utils.response import json_handler if separators is None: @@ -2044,7 +2044,7 @@ def are_emails_muted(): def get_test_records(doctype): - """Returns list of objects from `test_records.json` in the given doctype's folder.""" + """Return list of objects from `test_records.json` in the given doctype's folder.""" from frappe.modules import get_doctype_module, get_module_path path = os.path.join( @@ -2277,7 +2277,7 @@ log_level = None def logger( module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20 ): - """Returns a python logger that uses StreamHandler""" + """Return a python logger that uses StreamHandler.""" from frappe.utils.logger import get_logger return get_logger( @@ -2298,7 +2298,7 @@ def get_desk_link(doctype, name): def bold(text: str) -> str: - """Returns `text` wrapped in `` tags.""" + """Return `text` wrapped in `` tags.""" return f"{text}" @@ -2322,7 +2322,7 @@ def get_website_settings(key): def get_system_settings(key: str): - """Returns the value associated with the given `key` from System Settings DocType.""" + """Return the value associated with the given `key` from System Settings DocType.""" if not hasattr(local, "system_settings"): try: local.system_settings = get_cached_doc("System Settings") @@ -2341,7 +2341,7 @@ def get_active_domains(): def get_version(doctype, name, limit=None, head=False, raise_err=True): """ - Returns a list of version information of a given DocType. + Return a list of version information for the given DocType. Note: Applicable only if DocType has changes tracked. diff --git a/frappe/auth.py b/frappe/auth.py index 10bac8261a..56f1bcae26 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -289,7 +289,7 @@ class LoginManager: def check_password(self, user, pwd): """check password""" try: - # returns user in correct case + # return user in correct case return check_password(user, pwd) except frappe.AuthenticationError: self.fail("Incorrect password", user=user) diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index 5b7aa7818e..956bb0c9c3 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -297,11 +297,11 @@ class AutoRepeat(Document): def get_next_schedule_date(self, schedule_date, for_full_schedule=False): """ - Returns the next schedule date for auto repeat after a recurring document has been created. - Adds required offset to the schedule_date param and returns the next schedule date. + Return the next schedule date for auto repeat after a recurring document has been created. + Add required offset to the schedule_date param and return the next schedule date. :param schedule_date: The date when the last recurring document was created. - :param for_full_schedule: If True, returns the immediate next schedule date, else the full schedule. + :param for_full_schedule: If True, return the immediate next schedule date, else the full schedule. """ if month_map.get(self.frequency): month_count = month_map.get(self.frequency) + month_diff(schedule_date, self.start_date) - 1 diff --git a/frappe/build.py b/frappe/build.py index 7f111b9a69..8d66846b57 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -136,7 +136,7 @@ def setup_assets(assets_archive): def download_frappe_assets(verbose=True): """Downloads and sets up Frappe assets if they exist based on the current commit HEAD. - Returns True if correctly setup else returns False. + Return True if correctly setup else return False. """ frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD") @@ -407,7 +407,7 @@ def link_assets_dir(source, target, hard_link=False): def scrub_html_template(content): - """Returns HTML content with removed whitespace and comments""" + """Return HTML content with removed whitespace and comments.""" # remove whitespace to a single space content = WHITESPACE_PATTERN.sub(" ", content) @@ -418,7 +418,7 @@ def scrub_html_template(content): def html_to_js_template(path, content): - """returns HTML template content as Javascript code, adding it to `frappe.templates`""" + """Return HTML template content as Javascript code, by adding it to `frappe.templates`""" return """frappe.templates["{key}"] = '{content}';\n""".format( key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content) ) diff --git a/frappe/client.py b/frappe/client.py index c65753c766..2d7e765564 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -38,7 +38,7 @@ def get_list( as_dict: bool = True, or_filters=None, ): - """Returns a list of records by filters, fields, ordering and limit + """Return a list of records by filters, fields, ordering and limit. :param doctype: DocType of the data to be queried :param fields: fields to be returned. Default is `name` @@ -74,7 +74,7 @@ def get_count(doctype, filters=None, debug=False, cache=False): @frappe.whitelist() def get(doctype, name=None, filters=None, parent=None): - """Returns a document by name or filters + """Return a document by name or filters. :param doctype: DocType of the document to be returned :param name: return document of this `name` @@ -97,7 +97,7 @@ def get(doctype, name=None, filters=None, parent=None): @frappe.whitelist() def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, parent=None): - """Returns a value form a document + """Return a value from a document. :param doctype: DocType to be queried :param fieldname: Field to be returned (default `name`) @@ -296,7 +296,7 @@ def bulk_update(docs): @frappe.whitelist() def has_permission(doctype, docname, perm_type="read"): - """Returns a JSON with data whether the document has the requested permission + """Return a JSON with data whether the document has the requested permission. :param doctype: DocType of the document to be checked :param docname: `name` of the document to be checked @@ -307,7 +307,7 @@ def has_permission(doctype, docname, perm_type="read"): @frappe.whitelist() def get_doc_permissions(doctype, docname): - """Returns an evaluated document permissions dict like `{"read":1, "write":1}` + """Return an evaluated document permissions dict like `{"read":1, "write":1}`. :param doctype: DocType of the document to be evaluated :param docname: `name` of the document to be evaluated @@ -354,7 +354,7 @@ def get_js(items): @frappe.whitelist(allow_guest=True) def get_time_zone(): - """Returns default time zone""" + """Return the default time zone""" return {"time_zone": frappe.defaults.get_defaults().get("time_zone")} @@ -466,8 +466,7 @@ def validate_link(doctype: str, docname: str, fields=None): def insert_doc(doc) -> "Document": - """Inserts document and returns parent document object with appended child document - if `doc` is child document else returns the inserted document object + """Insert document and return parent document object with appended child document if `doc` is child document else return the inserted document object. :param doc: doc to insert (dict)""" diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 169c9eecb4..a008f4638c 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -143,7 +143,7 @@ def get_preferred_address(doctype, name, preferred_key="is_primary_address"): def get_default_address( doctype: str, name: str | None, sort_key: str = "is_primary_address" ) -> str | None: - """Returns default Address name for the given doctype, name""" + """Return default Address name for the given doctype, name.""" if sort_key not in ["is_shipping_address", "is_primary_address"]: return None @@ -228,7 +228,7 @@ def get_address_list(doctype, txt, filters, limit_start, limit_page_length=20, o def has_website_permission(doc, ptype, user, verbose=False): - """Returns true if there is a related lead or contact related to this document""" + """Return True if there is a related lead or contact related to this document.""" contact_name = frappe.db.get_value("Contact", {"email_id": frappe.session.user}) if contact_name: diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index 5eaf28def0..b40a27ca13 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -168,7 +168,7 @@ class Contact(Document): def get_default_contact(doctype, name): - """Returns default contact for the given doctype, name""" + """Return default contact for the given doctype, name.""" out = frappe.db.sql( """select parent, IFNULL((select is_primary_contact from tabContact c where c.name = dl.parent), 0) diff --git a/frappe/core/api/file.py b/frappe/core/api/file.py index aa8be30707..7756fd274b 100644 --- a/frappe/core/api/file.py +++ b/frappe/core/api/file.py @@ -15,8 +15,7 @@ def unzip_file(name: str): @frappe.whitelist() def get_attached_images(doctype: str, names: list[str] | str) -> frappe._dict: - """get list of image urls attached in form - returns {name: ['image.jpg', 'image.png']}""" + """Return list of image urls attached in form `{name: ['image.jpg', 'image.png']}`.""" if isinstance(names, str): names = json.loads(names) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 1422c446cd..0321cbe7eb 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -298,7 +298,7 @@ class Communication(Document, CommunicationEmailMixin): @staticmethod def _get_emails_list(emails=None, exclude_displayname=False): - """Returns list of emails from given email string. + """Return list of emails from given email string. * Removes duplicate mailids * Removes display name from email address if exclude_displayname is True @@ -309,15 +309,15 @@ class Communication(Document, CommunicationEmailMixin): return [email.lower() for email in set(emails) if email] def to_list(self, exclude_displayname=True): - """Returns to list.""" + """Return `to` list.""" return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname) def cc_list(self, exclude_displayname=True): - """Returns cc list.""" + """Return `cc` list.""" return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname) def bcc_list(self, exclude_displayname=True): - """Returns bcc list.""" + """Return `bcc` list.""" return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname) def get_attachments(self): @@ -615,9 +615,9 @@ def parse_email(email_strings): def get_email_without_link(email): - """ - returns email address without doctype links - returns admin@example.com for email admin+doctype+docname@example.com + """Return email address without doctype links. + + e.g. 'admin@example.com' is returned for email 'admin+doctype+docname@example.com' """ if not frappe.get_all("Email Account", filters={"enable_automatic_linking": 1}): return email diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 956fc97c60..2c05570cdb 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -29,7 +29,7 @@ class CommunicationEmailMixin: ) def get_email_with_displayname(self, email_address): - """Returns email address after adding displayname.""" + """Return email address after adding displayname.""" display_name, email = parse_addr(email_address) if display_name and display_name != email: return email_address @@ -151,7 +151,7 @@ class CommunicationEmailMixin: return self.content def get_attach_link(self, print_format): - """Returns public link for the attachment via `templates/emails/print_link.html`.""" + """Return public link for the attachment via `templates/emails/print_link.html`.""" return frappe.get_template("templates/emails/print_link.html").render( { "url": get_url(), diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 632a7b7e1d..84f6acf8af 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -514,8 +514,8 @@ class ImportFile: def parse_next_row_for_import(self, data): """ - Parses rows that make up a doc. A doc maybe built from a single row or multiple rows. - Returns the doc, rows, and data without the rows. + Parse rows that make up a doc. A doc maybe built from a single row or multiple rows. + Return the doc, rows, and data without the rows. """ doctypes = self.header.doctypes diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py index dc26c1f96f..01fa56a9ce 100644 --- a/frappe/core/doctype/docfield/docfield.py +++ b/frappe/core/doctype/docfield/docfield.py @@ -118,9 +118,10 @@ class DocField(Document): width: DF.Data | None # end: auto-generated types def get_link_doctype(self): - """Returns the Link doctype for the docfield (if applicable) - if fieldtype is Link: Returns "options" - if fieldtype is Table MultiSelect: Returns "options" of the Link field in the Child Table + """Return the Link doctype for the `docfield` (if applicable). + + * If fieldtype is Link: Return "options". + * If fieldtype is Table MultiSelect: Return "options" of the Link field in the Child Table. """ if self.fieldtype == "Link": return self.options diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 83b5395a4e..ba48805724 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -986,7 +986,7 @@ class DocType(Document): add_column(self.name, "parentfield", "Data") def get_max_idx(self): - """Returns the highest `idx`""" + """Return the highest `idx`.""" max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", self.name) return max_idx and max_idx[0][0] or 0 diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index f1a96bffd0..f6c0b1defa 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -544,7 +544,7 @@ class File(Document): return self._content def get_full_path(self): - """Returns file path from given file name""" + """Return file path using the set file name.""" file_path = self.file_url or self.file_name @@ -705,7 +705,7 @@ class File(Document): return has_permission(self, "read") def get_extension(self): - """returns split filename and extension""" + """Split and return filename and extension for the set `file_name`.""" return os.path.splitext(self.file_name) def create_attachment_record(self): diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 270ece6fa5..ce72220953 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -118,7 +118,7 @@ class Page(Document): shutil.rmtree(dir_path, ignore_errors=True) def is_permitted(self): - """Returns true if Has Role is not set or the user is allowed.""" + """Return True if `Has Role` is not set or the user is allowed.""" from frappe.utils import has_common allowed = [ diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index ad51b0415c..5fb0feeca2 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -105,7 +105,7 @@ class Report(Document): self.set("roles", roles) def is_permitted(self): - """Returns true if Has Role is not set or the user is allowed.""" + """Return True if `Has Role` is not set or the user is allowed.""" from frappe.utils import has_common allowed = [ diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index e19bdc681d..eaa8bc4b96 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -128,14 +128,14 @@ class ServerScript(Document): frappe.msgprint(str(e), title=_("Compilation warning")) def execute_method(self) -> dict: - """Specific to API endpoint Server Scripts + """Specific to API endpoint Server Scripts. - Raises: - frappe.DoesNotExistError: If self.script_type is not API - frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user + Raise: + frappe.DoesNotExistError: If self.script_type is not API. + frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user. - Returns: - dict: Evaluates self.script with frappe.utils.safe_exec.safe_exec and returns the flags set in it's safe globals + Return: + dict: Evaluate self.script with frappe.utils.safe_exec.safe_exec and return the flags set in it's safe globals. """ if self.enable_rate_limit: @@ -174,13 +174,13 @@ class ServerScript(Document): safe_exec(self.script, script_filename=self.name) def get_permission_query_conditions(self, user: str) -> list[str]: - """Specific to Permission Query Server Scripts + """Specific to Permission Query Server Scripts. Args: - user (str): Takes user email to execute script and return list of conditions + user (str): Take user email to execute script and return list of conditions. - Returns: - list: Returns list of conditions defined by rules in self.script + Return: + list: Return list of conditions defined by rules in self.script. """ locals = {"user": user, "conditions": ""} safe_exec(self.script, None, locals, script_filename=self.name) @@ -189,12 +189,10 @@ class ServerScript(Document): @frappe.whitelist() def get_autocompletion_items(self): - """Generates a list of a autocompletion strings from the context dict + """Generate a list of autocompletion strings from the context dict that is used while executing a Server Script. - Returns: - list: Returns list of autocompletion items. - For e.g., ["frappe.utils.cint", "frappe.get_all", ...] + e.g., ["frappe.utils.cint", "frappe.get_all", ...] """ def get_keys(obj): diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index 1a68368ba0..f2609ad05b 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -46,7 +46,7 @@ def validate_receiver_nos(receiver_list): @frappe.whitelist() def get_contact_number(contact_name, ref_doctype, ref_name): - "returns mobile number of the contact" + "Return mobile number of the given contact." number = frappe.db.sql( """select mobile_no, phone from tabContact where name=%s diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 028af756df..a56bfaaca0 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -230,7 +230,7 @@ class User(Document): frappe.cache.delete_key("users_for_mentions") def has_website_permission(self, ptype, user, verbose=False): - """Returns true if current user is the session user""" + """Return True if current user is the session user.""" return self.name == frappe.session.user def set_full_name(self): @@ -686,7 +686,7 @@ class User(Document): ) def get_blocked_modules(self): - """Returns list of modules blocked for that user""" + """Return list of modules blocked for that user.""" return [d.module for d in self.block_modules] if self.block_modules else [] def validate_user_email_inbox(self): @@ -1083,7 +1083,7 @@ def user_query(doctype, txt, searchfield, start, page_len, filters): def get_total_users(): - """Returns total no. of system users""" + """Return total no. of system users.""" return flt( frappe.db.sql( """SELECT SUM(`simultaneous_sessions`) @@ -1118,7 +1118,7 @@ def get_system_users(exclude_users: Iterable[str] | str | None = None, limit: in def get_active_users(): - """Returns No. of system users who logged in, in the last 3 days""" + """Return number of system users who logged in, in the last 3 days.""" return frappe.db.sql( """select count(*) from `tabUser` where enabled = 1 and user_type != 'Website User' @@ -1131,12 +1131,12 @@ def get_active_users(): def get_website_users(): - """Returns total no. of website users""" + """Return total no. of website users.""" return frappe.db.count("User", filters={"enabled": True, "user_type": "Website User"}) def get_active_website_users(): - """Returns No. of website users who logged in, in the last 3 days""" + """Return number of website users who logged in, in the last 3 days.""" return frappe.db.sql( """select count(*) from `tabUser` where enabled = 1 and user_type = 'Website User' diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index ea00b604c1..ae43eb2d9d 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -173,7 +173,7 @@ def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, def get_permitted_documents(doctype): - """Returns permitted documents from the given doctype for the session user""" + """Return permitted documents from the given doctype for the session user.""" # sort permissions in a way to make the first permission in the list to be default user_perm_list = sorted( get_user_permissions().get(doctype, []), key=lambda x: x.get("is_default"), reverse=True diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index 26e920bca9..928874912f 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -17,7 +17,7 @@ def get_notification_config(): def get_things_todo(as_list=False): - """Returns a count of incomplete todos""" + """Return a count of incomplete ToDos.""" data = frappe.get_list( "ToDo", fields=["name", "description"] if as_list else "count(*)", @@ -35,7 +35,7 @@ def get_things_todo(as_list=False): def get_todays_events(as_list: bool = False): - """Returns a count of todays events in calendar""" + """Return a count of today's events in calendar.""" from frappe.desk.doctype.event.event import get_events from frappe.utils import nowdate diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 1368ced6eb..71d6a4a002 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -109,8 +109,10 @@ def add(parent, role, permlevel): @frappe.whitelist() -def update(doctype, role, permlevel, ptype, value=None, if_owner=0): - """Update role permission params +def update( + doctype: str, role: str, permlevel: int, ptype: str, value=None, if_owner=0 +) -> str | None: + """Update role permission params. Args: doctype (str): Name of the DocType to update params for @@ -119,8 +121,8 @@ def update(doctype, role, permlevel, ptype, value=None, if_owner=0): ptype (str): permission type, example "read", "delete", etc. value (None, optional): value for ptype, None indicates False - Returns: - str: Refresh flag is permission is updated successfully + Return: + str: Refresh flag if permission is updated successfully """ def clear_cache(): diff --git a/frappe/core/utils.py b/frappe/core/utils.py index 93e09ce695..3e7fb8f350 100644 --- a/frappe/core/utils.py +++ b/frappe/core/utils.py @@ -7,7 +7,7 @@ import frappe def get_parent_doc(doc): - """Returns document of `reference_doctype`, `reference_doctype`""" + """Return document of `reference_doctype`, `reference_doctype`.""" if not hasattr(doc, "parent_doc"): if doc.reference_doctype and doc.reference_name: doc.parent_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name) @@ -38,8 +38,7 @@ def set_timeline_doc(doc): def find(list_of_dict, match_function): - """Returns a dict in a list of dicts on matching the conditions - provided in match function + """Return a dict in a list of dicts on matching the conditions provided in match function. Usage: list_of_dict = [{'name': 'Suraj'}, {'name': 'Aditya'}] @@ -54,8 +53,7 @@ def find(list_of_dict, match_function): def find_all(list_of_dict, match_function): - """Returns all matching dicts in a list of dicts. - Uses matching function to filter out the dicts + """Return all matching dicts in a list of dicts. Uses matching function to filter out the dicts. Usage: colored_shapes = [ @@ -87,6 +85,6 @@ def ljust_list(_list, length, fill_word=None): def html2text(html: str, strip_links=False, wrap=True) -> str: - """Returns the given `html` as markdown text.""" + """Return the given `html` as markdown text.""" strip = ["a"] if strip_links else None return md(html, heading_style="ATX", strip=strip, wrap=wrap) diff --git a/frappe/database/database.py b/frappe/database/database.py index d04135e827..ba25b41841 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -123,7 +123,7 @@ class Database: self._conn.select_db(db_name) def get_connection(self): - """Returns a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects""" + """Return a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects.""" raise NotImplementedError def get_database_size(self): @@ -160,7 +160,7 @@ class Database: :param ignore_ddl: Catch exception if table, column missing. :param auto_commit: Commit after executing the query. :param update: Update this dict to all rows (if returned `as_dict`). - :param run: Returns query without executing it if False. + :param run: Return query without executing it if False. :param pluck: Get the plucked field only. :param explain: Print `EXPLAIN` in error log. Examples: @@ -397,7 +397,7 @@ class Database: raise ImplicitCommitError("This statement can cause implicit commit", query) def fetch_as_dict(self) -> list[frappe._dict]: - """Internal. Converts results to dict.""" + """Internal. Convert results to dict.""" result = self.last_result if result: keys = [column[0] for column in self._cursor.description] @@ -410,7 +410,7 @@ class Database: frappe.cache.delete_key("db_tables") def get_description(self): - """Returns result metadata.""" + """Return result metadata.""" return self._cursor.description @staticmethod @@ -419,7 +419,7 @@ class Database: return [[value for value in row] for row in res] def get(self, doctype, filters=None, as_dict=True, cache=False): - """Returns `get_value` with fieldname='*'""" + """Return `get_value` with fieldname='*'.""" return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache) def get_value( @@ -438,7 +438,7 @@ class Database: pluck=False, distinct=False, ): - """Returns a document property or list of properties. + """Return a document property or list of properties. :param doctype: DocType name. :param filters: Filters like `{"x":"y"}` or name of the document. `None` if Single DocType. @@ -510,7 +510,7 @@ class Database: distinct=False, limit=None, ): - """Returns multiple document properties. + """Return multiple document properties. :param doctype: DocType name. :param filters: Filters like `{"x":"y"}` or name of the document. @@ -926,11 +926,11 @@ class Database: self.set_default(key, val, user) def get_global(self, key, user="__global"): - """Returns a global key value.""" + """Return a global key value.""" return self.get_default(key, user) def get_default(self, key, parent="__default"): - """Returns default value as a list if multiple or single""" + """Return default value as a list if multiple or single.""" d = self.get_defaults(key, parent) return isinstance(d, list) and d[0] or d @@ -1006,7 +1006,7 @@ class Database: return self.exists("DocField", {"fieldname": fn, "parent": dt}) def table_exists(self, doctype, cached=True): - """Returns True if table for given doctype exists.""" + """Return True if table for given doctype exists.""" return f"tab{doctype}" in self.get_tables(cached=cached) def has_table(self, doctype): @@ -1016,7 +1016,7 @@ class Database: raise NotImplementedError def a_row_exists(self, doctype): - """Returns True if atleast one row exists.""" + """Return True if at least one row exists.""" return frappe.get_all(doctype, limit=1, order_by=None, as_list=True) def exists(self, dt, dn=None, cache=False): @@ -1055,7 +1055,7 @@ class Database: return self.get_value(dt, dn, ignore=True, cache=cache, order_by=None) def count(self, dt, filters=None, debug=False, cache=False, distinct: bool = True): - """Returns `COUNT(*)` for given DocType and filters.""" + """Return `COUNT(*)` for given DocType and filters.""" if cache and not filters: cache_count = frappe.cache.get_value(f"doctype:count:{dt}") if cache_count is not None: @@ -1098,7 +1098,7 @@ class Database: ) def get_db_table_columns(self, table) -> list[str]: - """Returns list of column names from given table.""" + """Return list of column names from given table.""" columns = frappe.cache.hget("table_columns", table) if columns is None: information_schema = frappe.qb.Schema("information_schema") @@ -1116,14 +1116,14 @@ class Database: return columns def get_table_columns(self, doctype): - """Returns list of column names from given doctype.""" + """Return list of column names from given doctype.""" columns = self.get_db_table_columns("tab" + doctype) if not columns: raise self.TableMissingError("DocType", doctype) return columns def has_column(self, doctype, column): - """Returns True if column exists in database.""" + """Return True if column exists in database.""" return column in self.get_table_columns(doctype) def has_index(self, table_name, index_name): diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 1d2fc09120..1096821e66 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -191,7 +191,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): } def get_database_size(self): - """'Returns database size in MB""" + """Return database size in MB.""" db_size = self.sql( """ SELECT `table_schema` as `database_name`, @@ -314,7 +314,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): return "ON DUPLICATE key UPDATE " def get_table_columns_description(self, table_name): - """Returns list of column and its description""" + """Return list of columns with descriptions.""" return self.sql( """select column_name as 'name', @@ -339,7 +339,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): ) def get_column_type(self, doctype, column): - """Returns column type from database.""" + """Return column type from database.""" information_schema = frappe.qb.Schema("information_schema") table = get_table_name(doctype) @@ -446,7 +446,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): return self.sql("SHOW DATABASES", pluck=True) def get_tables(self, cached=True): - """Returns list of tables""" + """Return list of tables.""" to_query = not cached if cached: diff --git a/frappe/database/operator_map.py b/frappe/database/operator_map.py index d98f46d758..8f77db5687 100644 --- a/frappe/database/operator_map.py +++ b/frappe/database/operator_map.py @@ -17,20 +17,20 @@ def like(key: Field, value: str) -> frappe.qb: key (str): field value (str): criterion - Returns: + Return: frappe.qb: `frappe.qb object with `LIKE` """ return key.like(value) def func_in(key: Field, value: list | tuple) -> frappe.qb: - """Wrapper method for `IN` + """Wrapper method for `IN`. Args: key (str): field value (Union[int, str]): criterion - Returns: + Return: frappe.qb: `frappe.qb object with `IN` """ if isinstance(value, str): @@ -39,26 +39,26 @@ def func_in(key: Field, value: list | tuple) -> frappe.qb: def not_like(key: Field, value: str) -> frappe.qb: - """Wrapper method for `NOT LIKE` + """Wrapper method for `NOT LIKE`. Args: key (str): field value (str): criterion - Returns: + Return: frappe.qb: `frappe.qb object with `NOT LIKE` """ return key.not_like(value) def func_not_in(key: Field, value: list | tuple | str): - """Wrapper method for `NOT IN` + """Wrapper method for `NOT IN`. Args: key (str): field value (Union[int, str]): criterion - Returns: + Return: frappe.qb: `frappe.qb object with `NOT IN` """ if isinstance(value, str): @@ -73,20 +73,20 @@ def func_regex(key: Field, value: str) -> frappe.qb: key (str): field value (str): criterion - Returns: + Return: frappe.qb: `frappe.qb object with `REGEX` """ return key.regex(value) def func_between(key: Field, value: list | tuple) -> frappe.qb: - """Wrapper method for `BETWEEN` + """Wrapper method for `BETWEEN`. Args: key (str): field value (Union[int, str]): criterion - Returns: + Return: frappe.qb: `frappe.qb object with `BETWEEN` """ return key[slice(*value)] @@ -98,13 +98,13 @@ def func_is(key, value): def func_timespan(key: Field, value: str) -> frappe.qb: - """Wrapper method for `TIMESPAN` + """Wrapper method for `TIMESPAN`. Args: key (str): field value (str): criterion - Returns: + Return: frappe.qb: `frappe.qb object with `TIMESPAN` """ diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 56a1b8733f..617350cca5 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -197,7 +197,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): return str(psycopg2.extensions.QuotedString(s)) def get_database_size(self): - """'Returns database size in MB""" + """Return database size in MB""" db_size = self.sql( "SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", self.db_name, as_dict=True ) @@ -380,7 +380,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): ) def get_table_columns_description(self, table_name): - """Returns list of column and its description""" + """Return list of columns with description.""" # pylint: disable=W1401 return self.sql( """ @@ -411,7 +411,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): ) def get_column_type(self, doctype, column): - """Returns column type from database.""" + """Return column type from database.""" information_schema = frappe.qb.Schema("information_schema") table = get_table_name(doctype) diff --git a/frappe/database/query.py b/frappe/database/query.py index 06295d33a6..a2cb2486f4 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -218,7 +218,7 @@ class Engine: self.query = self.query.where(operator_fn(_field, _value)) def get_function_object(self, field: str) -> "Function": - """Expects field to look like 'SUM(*)' or 'name' or something similar. Returns PyPika Function object""" + """Return PyPika Function object. Expect field to look like 'SUM(*)' or 'name' or something similar.""" func = field.split("(", maxsplit=1)[0].capitalize() args_start, args_end = len(func) + 1, field.index(")") args = field[args_start:args_end].split(",") diff --git a/frappe/defaults.py b/frappe/defaults.py index 65b145f338..d8ba0dc93b 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -79,7 +79,7 @@ def is_a_user_permission_key(key): def not_in_user_permission(key, value, user=None): - # returns true or false based on if value exist in user permission + # return true or false based on if value exist in user permission user = user or frappe.session.user user_permission = get_user_permissions(user).get(frappe.unscrub(key)) or [] diff --git a/frappe/desk/calendar.py b/frappe/desk/calendar.py index d8c058536d..478c53c395 100644 --- a/frappe/desk/calendar.py +++ b/frappe/desk/calendar.py @@ -19,7 +19,7 @@ def update_event(args, field_map): def get_event_conditions(doctype, filters=None): - """Returns SQL conditions with user permissions and filters for event queries""" + """Return SQL conditions with user permissions and filters for event queries.""" from frappe.desk.reportview import get_filters_cond if not frappe.has_permission(doctype): diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 8ae20f7bb0..a7c9a5ef0c 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -68,7 +68,7 @@ class Workspace: ) def is_permitted(self): - """Returns true if Has Role is not set or the user is allowed.""" + """Return true if `Has Role` is not set or the user is allowed.""" from frappe.utils import has_common allowed = [d.role for d in self.doc.roles] @@ -383,13 +383,12 @@ class Workspace: @frappe.whitelist() @frappe.read_only() def get_desktop_page(page): - """Applies permissions, customizations and returns the configruration for a page - on desk. + """Apply permissions, customizations and return the configuration for a page on desk. Args: page (json): page data - Returns: + Return: dict: dictionary of cards, charts and shortcuts to be displayed on website """ try: @@ -503,7 +502,7 @@ def get_custom_doctype_list(module): def get_custom_report_list(module): - """Returns list on new style reports for modules.""" + """Return list on new style reports for modules.""" reports = frappe.get_all( "Report", fields=["name", "ref_doctype", "report_type"], @@ -617,14 +616,14 @@ def new_widget(config, doctype, parentfield): def prepare_widget(config, doctype, parentfield): - """Create widget child table entries with parent details + """Create widget child table entries with parent details. Args: config (dict): Dictionary containing widget config doctype (string): Doctype name of the child table parentfield (string): Parent field for the child table - Returns: + Return: TYPE: List of Document objects """ if not config: diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 65ddf5cde4..f71afef6da 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -77,15 +77,15 @@ class DocTags: self.dt = dt def get_tag_fields(self): - """returns tag_fields property""" + """Return `tag_fields` property.""" return frappe.db.get_value("DocType", self.dt, "tag_fields") def get_tags(self, dn): - """returns tag for a particular item""" + """Return tag for a particular item.""" return (frappe.db.get_value(self.dt, dn, "_user_tags", ignore=1) or "").strip() def add(self, dn, tag): - """add a new user tag""" + """Add a new user tag.""" tl = self.get_tags(dn).split(",") if tag not in tl: tl.append(tag) @@ -94,16 +94,16 @@ class DocTags: self.update(dn, tl) def remove(self, dn, tag): - """remove a user tag""" + """Remove a user tag.""" tl = self.get_tags(dn).split(",") self.update(dn, filter(lambda x: x.lower() != tag.lower(), tl)) def remove_all(self, dn): - """remove all user tags (call before delete)""" + """Remove all user tags (call before delete).""" self.update(dn, []) def update(self, dn, tl): - """updates the _user_tag column in the table""" + """Update the `_user_tag` column in the table.""" if not tl: tags = "" @@ -128,16 +128,15 @@ class DocTags: raise def setup(self): - """adds the _user_tags column if not exists""" + """Add the `_user_tags` column if not exists.""" from frappe.database.schema import add_column add_column(self.dt, "_user_tags", "Data") def delete_tags_for_document(doc): - """ - Delete the Tag Link entry of a document that has - been deleted + """Delete the Tag Link entry of a document that has been deleted. + :param doc: Deleted document """ if not frappe.db.table_exists("Tag Link"): @@ -147,7 +146,7 @@ def delete_tags_for_document(doc): def update_tags(doc, tags): - """Adds tags for documents + """Add tags for documents. :param doc: Document to be added to global tags """ @@ -181,8 +180,8 @@ def update_tags(doc, tags): @frappe.whitelist() def get_documents_for_tag(tag): - """ - Search for given text in Tag Link + """Search for given text in Tag Link. + :param tag: tag to be searched """ # remove hastag `#` from tag diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 858ec2c5b8..d6427f9388 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -136,7 +136,7 @@ class ToDo(Document): @classmethod def get_owners(cls, filters=None): - """Returns list of owners after applying filters on todo's.""" + """Return list of owners after applying filters on ToDos.""" rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=["allocated_to"]) return [parse_addr(row.allocated_to)[1] for row in rows if row.allocated_to] diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index cbe6dd7acc..45bf42fd8f 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -131,7 +131,7 @@ class SubmittableDocumentTree: return self._references_across_doctypes.get(doctype, []) def get_document_sources(self): - """Returns list of doctypes from where we access submittable documents.""" + """Return list of doctypes from where we access submittable documents.""" return list(set(self.get_link_sources() + [self.root_doctype])) def get_link_sources(self): @@ -139,7 +139,7 @@ class SubmittableDocumentTree: return list(set(self.get_submittable_doctypes()) - set(get_exempted_doctypes() or [])) def get_submittable_doctypes(self) -> list[str]: - """Returns list of submittable doctypes.""" + """Return list of submittable doctypes.""" if not self._submittable_doctypes: self._submittable_doctypes = frappe.get_all( "DocType", {"is_submittable": 1}, pluck="name", order_by=None @@ -148,7 +148,7 @@ class SubmittableDocumentTree: def get_child_tables_of_doctypes(doctypes: list[str] = None): - """Returns child tables by doctype.""" + """Return child tables by doctype.""" filters = [["fieldtype", "=", "Table"]] filters_for_docfield = filters filters_for_customfield = filters @@ -387,7 +387,7 @@ def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=None): docinfo (dict): The document to check for submitted and non-exempt from auto-cancel ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling. - Returns: + Return: bool: True if linked document passes all validations, else False """ # ignore doctype to cancel diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 993899f4bf..cc515c0ff1 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -274,7 +274,7 @@ def _get_communications(doctype, name, start=0, limit=20): def get_communication_data( doctype, name, start=0, limit=20, after=None, fields=None, group_by=None, as_dict=True ): - """Returns list of communications for a given document""" + """Return list of communications for a given document.""" if not fields: fields = """ C.name, C.communication_type, C.communication_medium, diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 7ca483d806..14c4d1c13f 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -123,7 +123,7 @@ def generate_report_result( def normalize_result(result, columns): - # Converts to list of dicts from list of lists/tuples + # Convert to list of dicts from list of lists/tuples data = [] column_names = [column["fieldname"] for column in columns] if result and isinstance(result[0], (list, tuple)): @@ -603,11 +603,11 @@ def has_match( columns_dict, user, ): - """Returns True if after evaluating permissions for each linked doctype - - There is an owner match for the ref_doctype - - `and` There is a user permission match for all linked doctypes + """Return True if after evaluating permissions for each linked doctype: + - There is an owner match for the ref_doctype + - `and` There is a user permission match for all linked doctypes - Returns True if the row is empty + Return True if the row is empty. Note: Each doctype could have multiple conflicting user permission doctypes. @@ -705,9 +705,10 @@ def get_linked_doctypes(columns, data): def get_columns_dict(columns): - """Returns a dict with column docfield values as dict + """Return a dict with column docfield values as dict. + The keys for the dict are both idx and fieldname, - so either index or fieldname can be used to search for a column's docfield properties + so either index or fieldname can be used to search for a column's docfield properties. """ columns_dict = frappe._dict() for idx, col in enumerate(columns): diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index bbede90a34..6fe2596d7f 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -128,7 +128,7 @@ class AutoEmailReport(Document): ) def get_report_content(self): - """Returns file in for the report in given format""" + """Return file for the report in given format.""" report = frappe.get_doc("Report", self.report) self.filters = frappe.parse_json(self.filters) if self.filters else {} diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 6df8a4287c..2f4c23eac6 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -237,7 +237,7 @@ class EmailAccount(Document): return frappe.db.get_value("Email Domain", domain, EMAIL_DOMAIN_FIELDS, as_dict=True) def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): - """Returns logged in POP3/IMAP connection object.""" + """Return logged in POP3/IMAP connection object.""" oauth_token = self.get_oauth_token() args = frappe._dict( { diff --git a/frappe/email/doctype/email_template/email_template.py b/frappe/email/doctype/email_template/email_template.py index 214879dac4..5f75589eff 100644 --- a/frappe/email/doctype/email_template/email_template.py +++ b/frappe/email/doctype/email_template/email_template.py @@ -48,7 +48,7 @@ class EmailTemplate(Document): @frappe.whitelist() def get_email_template(template_name, doc): - """Returns the processed HTML of a email template with the given doc""" + """Return the processed HTML of a email template with the given doc""" email_template = frappe.get_doc("Email Template", template_name) return email_template.get_formatted_email(doc) diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 268de161b3..449c0b5b15 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -394,7 +394,7 @@ def get_email_html(template, args, subject, header=None, with_container=False): def inline_style_in_html(html): - """Convert email.css and html to inline-styled html""" + """Convert email.css and html to inline-styled html.""" from premailer import Premailer from frappe.utils.jinja_globals import bundled_asset @@ -460,7 +460,7 @@ def add_attachment(fname, fcontent, content_type=None, parent=None, content_id=N def get_message_id(): - """Returns Message ID created from doctype and name""" + """Return Message ID created from doctype and name.""" return email.utils.make_msgid(domain=frappe.local.site) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 3083cdb264..72a2dfce82 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -162,7 +162,7 @@ class EmailServer: return def get_messages(self, folder="INBOX"): - """Returns new email messages.""" + """Return new email messages.""" self.latest_messages = [] self.seen_status = {} @@ -864,7 +864,7 @@ class InboundMail(Email): @staticmethod def get_email_fields(doctype): - """Returns Email related fields of a doctype.""" + """Return Email related fields of a doctype.""" fields = frappe._dict() email_fields = ["subject_field", "sender_field", "sender_name_field"] diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py index 7ad016828a..0e18fbf483 100644 --- a/frappe/frappeclient.py +++ b/frappe/frappeclient.py @@ -109,7 +109,7 @@ class FrappeClient: def get_list( self, doctype, fields='["name"]', filters=None, limit_start=0, limit_page_length=None ): - """Returns list of records of a particular type""" + """Return list of records of a particular type.""" if not isinstance(fields, str): fields = json.dumps(fields) params = { @@ -173,7 +173,7 @@ class FrappeClient: return self.post_request({"cmd": "frappe.client.submit", "doc": frappe.as_json(doc)}) def get_value(self, doctype, fieldname=None, filters=None): - """Returns a value form a document + """Return a value from a document. :param doctype: DocType to be queried :param fieldname: Field to be returned (default `name`) @@ -212,7 +212,7 @@ class FrappeClient: return self.post_request({"cmd": "frappe.client.cancel", "doctype": doctype, "name": name}) def get_doc(self, doctype, name="", filters=None, fields=None): - """Returns a single remote document + """Return a single remote document. :param doctype: DocType of the document to be returned :param name: (optional) `name` of the document to be returned diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py index 662e058d68..9b4a42179c 100644 --- a/frappe/geo/utils.py +++ b/frappe/geo/utils.py @@ -19,7 +19,7 @@ def get_coords(doctype, filters, type): def convert_to_geojson(type, coords): - """Converts GPS coordinates to geoJSON string.""" + """Convert GPS coordinates to geoJSON string.""" geojson = {"type": "FeatureCollection", "features": None} if type == "location_field": @@ -90,7 +90,7 @@ def return_coordinates(doctype, filters_sql): def get_coords_conditions(doctype, filters=None): - """Returns SQL conditions with user permissions and filters for event queries.""" + """Return SQL conditions with user permissions and filters for event queries.""" from frappe.desk.reportview import get_filters_cond if not frappe.has_permission(doctype): diff --git a/frappe/installer.py b/frappe/installer.py index c271608833..891de5f2e8 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -419,7 +419,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]: Note: All record linked linked to Module Def are also deleted. - Returns: list of deleted doctypes.""" + Return: list of deleted doctypes.""" drop_doctypes = [] doctype_link_field_map = _get_module_linked_doctype_field_map() @@ -461,7 +461,7 @@ def _delete_linked_documents( def _get_module_linked_doctype_field_map() -> dict[str, str]: """Get all the doctypes which have module linked with them. - returns ordered dictionary with doctype->link field mapping.""" + Return ordered dictionary with doctype->link field mapping.""" # Hardcoded to change order of deletion ordered_doctypes = [ @@ -672,7 +672,7 @@ def extract_sql_from_archive(sql_file_path): Args: sql_file_path (str): Path of the SQL file - Returns: + Return: str: Path of the decompressed SQL file """ from frappe.utils import get_bench_relative_path diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index 3f412efc90..8430e5c80c 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -202,9 +202,7 @@ def sync(g_calendar=None): def get_google_calendar_object(g_calendar): - """ - Returns an object of Google Calendar along with Google Calendar doc. - """ + """Return an object of Google Calendar along with Google Calendar doc.""" google_settings = frappe.get_doc("Google Settings") account = frappe.get_doc("Google Calendar", g_calendar) @@ -257,8 +255,8 @@ def check_google_calendar(account, google_calendar): def sync_events_from_google_calendar(g_calendar, method=None): - """ - Syncs Events from Google Calendar in Framework Calendar. + """Sync Events from Google Calendar in Framework Calendar. + Google Calendar returns nextSyncToken when all the events in Google Calendar are fetched. nextSyncToken is returned at the very last page https://developers.google.com/calendar/v3/sync @@ -685,12 +683,10 @@ def format_date_according_to_google_calendar(all_day, starts_on, ends_on=None): def parse_google_calendar_recurrence_rule(repeat_day_week_number, repeat_day_name): - """ - Returns (repeat_on) exact date for combination eg 4TH viz. 4th thursday of a month - """ + """Return (repeat_on) exact date for combination eg 4TH viz. 4th thursday of a month.""" if repeat_day_week_number < 0: # Consider a month with 5 weeks and event is to be repeated in last week of every month, google caledar considers - # a month has 4 weeks and hence itll return -1 for a month with 5 weeks. + # a month has 4 weeks and hence it'll return -1 for a month with 5 weeks. repeat_day_week_number = 4 weekdays = get_weekdays() @@ -714,9 +710,7 @@ def parse_google_calendar_recurrence_rule(repeat_day_week_number, repeat_day_nam def repeat_on_to_google_calendar_recurrence_rule(doc): - """ - Returns event (repeat_on) in Google Calendar format ie RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH - """ + """Return event (repeat_on) in Google Calendar format ie RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH.""" recurrence = framework_frequencies.get(doc.repeat_on) weekdays = get_weekdays() @@ -732,8 +726,8 @@ def repeat_on_to_google_calendar_recurrence_rule(doc): def get_week_number(dt): - """ - Returns the week number of the month for the specified date. + """Return the week number of the month for the specified date. + https://stackoverflow.com/questions/3806473/python-week-number-of-the-month/16804556 """ from math import ceil @@ -771,9 +765,7 @@ def get_conference_data(doc): def get_attendees(doc): - """ - Returns a list of dicts with attendee emails, if available in event_participants table - """ + """Return a list of dicts with attendee emails, if available in event_participants table.""" attendees, email_not_found = [], [] for participant in doc.event_participants: diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index 65bd2bf1d3..fa316de026 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -74,9 +74,7 @@ def authorize_access(g_contact, reauthorize=False, code=None): def get_google_contacts_object(g_contact): - """ - Returns an object of Google Calendar along with Google Calendar doc. - """ + """Return an object of Google Calendar along with Google Calendar doc.""" account = frappe.get_doc("Google Contacts", g_contact) oauth_obj = GoogleOAuth("contacts") diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index aa85bac06d..a8c44796ef 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -88,9 +88,7 @@ def authorize_access(reauthorize=False, code=None): def get_google_drive_object(): - """ - Returns an object of Google Drive. - """ + """Return an object of Google Drive.""" account = frappe.get_doc("Google Drive") oauth_obj = GoogleOAuth("drive") diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py index 63eadd7f4a..b66f7e9479 100644 --- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py +++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py @@ -21,7 +21,7 @@ class OAuthProviderSettings(Document): def get_oauth_settings(): - """Returns oauth settings""" + """Return OAuth settings.""" return frappe._dict( { "skip_authorization": frappe.db.get_single_value( diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index 8bc54e0b1d..7f24c611bf 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -56,7 +56,7 @@ class GoogleOAuth: frappe.throw(frappe._("Please update {} before continuing.").format(google_settings)) def authorize(self, oauth_code: str) -> dict[str, str | int]: - """Returns a dict with access and refresh token. + """Return a dict with access and refresh token. :param oauth_code: code got back from google upon successful auhtorization """ @@ -99,7 +99,7 @@ class GoogleOAuth: ) def get_authentication_url(self, state: dict[str, str]) -> dict[str, str]: - """Returns google authentication url. + """Return Google authentication url. :param state: dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc) """ @@ -117,7 +117,7 @@ class GoogleOAuth: } def get_google_service_object(self, access_token: str, refresh_token: str): - """Returns google service object""" + """Return Google service object.""" credentials_dict = { "token": access_token, diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index 2cafc8ba61..14ae944192 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -34,7 +34,7 @@ def make_request( def make_get_request(url: str, **kwargs): - """Makes a 'GET' HTTP request to the given `url` and returns processed response. + """Make a 'GET' HTTP request to the given `url` and return processed response. You can optionally pass the below parameters: @@ -46,7 +46,7 @@ def make_get_request(url: str, **kwargs): def make_post_request(url: str, **kwargs): - """Makes a 'POST' HTTP request to the given `url` and returns processed response. + """Make a 'POST' HTTP request to the given `url` and return processed response. You can optionally pass the below parameters: @@ -60,7 +60,7 @@ def make_post_request(url: str, **kwargs): def make_put_request(url: str, **kwargs): - """Makes a 'PUT' HTTP request to the given `url` and returns processed response. + """Make a 'PUT' HTTP request to the given `url` and return processed response. You can optionally pass the below parameters: @@ -74,7 +74,7 @@ def make_put_request(url: str, **kwargs): def make_patch_request(url: str, **kwargs): - """Makes a 'PATCH' HTTP request to the given `url` and returns processed response. + """Make a 'PATCH' HTTP request to the given `url` and return processed response. You can optionally pass the below parameters: @@ -88,7 +88,7 @@ def make_patch_request(url: str, **kwargs): def make_delete_request(url: str, **kwargs): - """Makes a 'DELETE' HTTP request to the given `url` and returns processed response. + """Make a 'DELETE' HTTP request to the given `url` and return processed response. You can optionally pass the below parameters: diff --git a/frappe/migrate.py b/frappe/migrate.py index 33c930e9da..cad55d5d24 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -151,7 +151,7 @@ class SiteMigration: frappe.get_attr(fn)() def required_services_running(self) -> bool: - """Returns True if all required services are running. Returns False and prints + """Return True if all required services are running. Return False and print instructions to stdout when required services are not available. """ service_status = check_connection(redis_services=["redis_cache"]) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 640de01216..241bfdee31 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -55,9 +55,9 @@ DOCTYPES_FOR_DOCTYPE = {"DocType", *TABLE_DOCTYPES_FOR_DOCTYPE.values()} def get_controller(doctype): - """ - Returns the locally cached **class** object of the given DocType. - For `custom` type, returns `frappe.model.document.Document`. + """Return the locally cached **class** object of the given DocType. + + For `custom` type, return `frappe.model.document.Document`. :param doctype: DocType name as string. """ @@ -146,9 +146,9 @@ class BaseDocument: return get_permitted_fields(doctype=self.doctype, parenttype=getattr(self, "parenttype", None)) def __getstate__(self): - """ + """Return a copy of `__dict__` excluding unpicklable values like `meta`. + Called when pickling. - Returns a copy of `__dict__` excluding unpicklable values like `meta`. More info: https://docs.python.org/3/library/pickle.html#handling-stateful-objects """ @@ -633,7 +633,7 @@ class BaseDocument: def get_field_name_by_key_name(self, key_name): """MariaDB stores a mapping between `key_name` and `column_name`. - This function returns the `column_name` associated with the `key_name` passed + Return the `column_name` associated with the `key_name` passed. Args: key_name (str): The name of the database index. @@ -641,7 +641,7 @@ class BaseDocument: Raises: IndexError: If the key is not found in the table. - Returns: + Return: str: The column name associated with the key. """ return frappe.db.sql( @@ -660,12 +660,12 @@ class BaseDocument: )[0].get("Column_name") def get_label_from_fieldname(self, fieldname): - """Returns the associated label for fieldname + """Return the associated label for fieldname. Args: fieldname (str): The fieldname in the DocType to use to pull the label. - Returns: + Return: str: The label associated with the fieldname, if found, otherwise `None`. """ df = self.meta.get_field(fieldname) @@ -743,7 +743,7 @@ class BaseDocument: return missing def get_invalid_links(self, is_submittable=False): - """Returns list of invalid links and also updates fetch values if not set""" + """Return list of invalid links and also update fetch values if not set.""" def get_msg(df, docname): # check if parentfield exists (only applicable for child table doctype) @@ -1112,7 +1112,7 @@ class BaseDocument: return "".join(set(pwd)) == "*" def precision(self, fieldname, parentfield=None) -> int | None: - """Returns float precision for a particular field (or get global default). + """Return float precision for a particular field (or get global default). :param fieldname: Fieldname for which precision is required. :param parentfield: If fieldname is in child table.""" @@ -1174,7 +1174,7 @@ class BaseDocument: return format_value(val, df=df, doc=doc, currency=currency, format=format) def is_print_hide(self, fieldname, df=None, for_print=True): - """Returns true if fieldname is to be hidden for print. + """Return True if fieldname is to be hidden for print. Print Hide can be set via the Print Format Builder or in the controller as a list of hidden fields. Example @@ -1203,7 +1203,8 @@ class BaseDocument: return print_hide def in_format_data(self, fieldname): - """Returns True if shown via Print Format::`format_data` property. + """Return True if shown via Print Format::`format_data` property. + Called from within standard print format.""" doc = getattr(self, "parent_doc", self) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 70ecedd3c8..3f459c8533 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -335,7 +335,7 @@ class DatabaseQuery: return args def parse_args(self): - """Convert fields and filters from strings to list, dicts""" + """Convert fields and filters from strings to list, dicts.""" if isinstance(self.fields, str): if self.fields == "*": self.fields = ["*"] @@ -715,7 +715,8 @@ class DatabaseQuery: j = j + len(permitted_fields) - 1 def prepare_filter_condition(self, f): - """Returns a filter condition in the format: + """Return a filter condition in the format: + ifnull(`tabDocType`.`fieldname`, fallback) operator "value" """ @@ -1339,7 +1340,7 @@ def get_date_range(operator: str, value: str): def requires_owner_constraint(role_permissions): - """Returns True if "select" or "read" isn't available without being creator.""" + """Return True if "select" or "read" isn't available without being creator.""" if not role_permissions.get("has_if_owner_enabled"): return diff --git a/frappe/model/document.py b/frappe/model/document.py index da0f2aca87..9135b07dc5 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: def get_doc(*args, **kwargs): - """returns a frappe.model.Document object. + """Return a `frappe.model.Document` object. :param arg1: Document dict or DocType name. :param arg2: [optional] document name. @@ -454,7 +454,7 @@ class Document(BaseDocument): return getattr(self, "_doc_before_save", None) def has_value_changed(self, fieldname): - """Returns true if value is changed before and after saving""" + """Return True if value has changed before and after saving.""" previous = self.get_doc_before_save() return previous.get(fieldname) != self.get(fieldname) if previous else True @@ -924,7 +924,7 @@ class Document(BaseDocument): frappe.throw(_("Cannot link cancelled document: {0}").format(msg), frappe.CancelledLinkError) def get_all_children(self, parenttype=None) -> list["Document"]: - """Returns all children documents from **Table** type fields in a list.""" + """Return all children documents from **Table** type fields in a list.""" children = [] @@ -977,7 +977,7 @@ class Document(BaseDocument): if self.flags.notifications is None: def _get_notifications(): - """returns enabled notifications for the current doctype""" + """Return enabled notifications for the current doctype.""" return frappe.get_all( "Notification", @@ -1379,7 +1379,7 @@ class Document(BaseDocument): doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield")))) def get_url(self): - """Returns Desk URL for this document.""" + """Return Desk URL for this document.""" return get_absolute_url(self.doctype, self.name) def add_comment( @@ -1453,7 +1453,7 @@ class Document(BaseDocument): ) def get_signature(self): - """Returns signature (hash) for private URL.""" + """Return signature (hash) for private URL.""" return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest() def get_document_share_key(self, expires_on=None, no_expiry=False): diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index f6b710064d..d1799682ed 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -10,8 +10,8 @@ from frappe.utils import cstr @frappe.whitelist() def make_mapped_doc(method, source_name, selected_children=None, args=None): - """Returns the mapped document calling the given mapper method. - Sets selected_children as flags for the `get_mapped_doc` method. + """Return the mapped document calling the given mapper method. + Set `selected_children` as flags for the `get_mapped_doc` method. Called from `open_mapped_doc` from create_new.js""" @@ -38,11 +38,12 @@ def make_mapped_doc(method, source_name, selected_children=None, args=None): @frappe.whitelist() def map_docs(method, source_names, target_doc, args=None): - '''Returns the mapped document calling the given mapper method - with each of the given source docs on the target doc + """Return the mapped document calling the given mapper method with each of the given source docs on the target doc. :param args: Args as string to pass to the mapper method - E.g. args: "{ 'supplier': 'XYZ' }"''' + + e.g. args: "{ 'supplier': 'XYZ' }" + """ method = frappe.get_attr(method) if method not in frappe.whitelisted: diff --git a/frappe/model/meta.py b/frappe/model/meta.py index df04dc1eda..0373ed81f3 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -208,7 +208,7 @@ class Meta(Document): return self._table_fields def get_global_search_fields(self): - """Returns list of fields with `in_global_search` set and `name` if set""" + """Return list of fields with `in_global_search` set and `name` if set.""" fields = self.get("fields", {"in_global_search": 1, "fieldtype": ["not in", no_value_fields]}) if getattr(self, "show_name_in_global_search", None): fields.append(frappe._dict(fieldtype="Data", fieldname="name", label="Name")) @@ -233,17 +233,17 @@ class Meta(Document): return TABLE_DOCTYPES_FOR_DOCTYPE.get(fieldname) def get_field(self, fieldname): - """Return docfield from meta""" + """Return docfield from meta.""" return self._fields.get(fieldname) def has_field(self, fieldname): - """Returns True if fieldname exists""" + """Return True if fieldname exists.""" return fieldname in self._fields def get_label(self, fieldname): - """Get label of the given fieldname""" + """Return label of the given fieldname.""" if df := self.get_field(fieldname): return df.get("label") @@ -273,8 +273,8 @@ class Meta(Document): return search_fields def get_fields_to_fetch(self, link_fieldname=None): - """Returns a list of docfield objects for fields whose values - are to be fetched and updated for a particular link field + """Return a list of docfield objects for fields whose values + are to be fetched and updated for a particular link field. These fields are of type Data, Link, Text, Readonly and their fetch_from property is set as `link_fieldname`.`source_fieldname`""" @@ -615,7 +615,7 @@ class Meta(Document): return permissions def get_dashboard_data(self): - """Returns dashboard setup related to this doctype. + """Return dashboard setup related to this doctype. This method will return the `data` property in the `[doctype]_dashboard.py` file in the doctype's folder, along with any overrides or extensions @@ -695,7 +695,7 @@ class Meta(Document): return self.get_web_template(suffix="_list") def get_web_template(self, suffix=""): - """Returns the relative path of the row template for this doctype""" + """Return the relative path of the row template for this doctype.""" module_name = frappe.scrub(self.module) doctype = frappe.scrub(self.name) template_path = frappe.get_module_path( diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 7a06e4d248..e775e0573b 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -334,7 +334,7 @@ def parse_naming_series( def has_custom_parser(e): - """Returns true if the naming series part has a custom parser""" + """Return True if the naming series part has a custom parser.""" return frappe.get_hooks("naming_series_variables", {}).get(e) diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index b4d30a5943..ed7e529e4a 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -30,8 +30,7 @@ def update_document_title( **kwargs, ) -> str: """ - Update the name or title of a document. Returns `name` if document was renamed, - `docname` if renaming operation was queued. + Update the name or title of a document. Return `name` if document was renamed, `docname` if renaming operation was queued. :param doctype: DocType of the document :param docname: Name of the document diff --git a/frappe/model/utils/__init__.py b/frappe/model/utils/__init__.py index f8f5b21de4..153a42ec12 100644 --- a/frappe/model/utils/__init__.py +++ b/frappe/model/utils/__init__.py @@ -84,7 +84,7 @@ def render_include(content): def get_fetch_values(doctype, fieldname, value): - """Returns fetch value dict for the given object + """Return fetch value dict for the given object. :param doctype: Target doctype :param fieldname: Link fieldname selected diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index ee8b4cd014..b58bdf235f 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -12,13 +12,10 @@ from frappe.utils import get_datetime, now def calculate_hash(path: str) -> str: - """Calculate md5 hash of the file in binary mode + """Calculate and return md5 hash of the file in binary mode. Args: path (str): Path to the file to be hashed - - Returns: - str: The calculated hash """ hash_md5 = hashlib.md5(usedforsecurity=False) with open(path, "rb") as f: @@ -82,8 +79,8 @@ def import_file_by_path( pre_process=None, ignore_version: bool = None, reset_permissions: bool = False, -): - """Import file from the given path +) -> bool: + """Import file from the given path. Some conditions decide if a file should be imported or not. Evaluation takes place in the order they are mentioned below. @@ -107,8 +104,7 @@ def import_file_by_path( ignore_version (bool, optional): ignore current version. Defaults to None. reset_permissions (bool, optional): reset permissions for the file. Defaults to False. - Returns: - [bool]: True if import takes place. False if it wasn't imported. + Return True if import takes place, False if it wasn't imported. """ try: docs = read_doc_from_file(path) diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index 40e3b32690..5d4cd6deac 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -22,10 +22,9 @@ doctype_python_modules = {} def export_module_json(doc: "Document", is_standard: bool, module: str) -> str | None: - """Make a folder for the given doc and add its json file (make it a standard - object that will be synced) + """Make a folder for the given doc and add its json file (make it a standard object that will be synced). - Returns the absolute file_path without the extension. + Return the absolute file_path without the extension. Eg: For exporting a Print Format "_Test Print Format 1", the return value will be `/home/gavin/frappe-bench/apps/frappe/frappe/core/print_format/_test_print_format_1/_test_print_format_1` """ @@ -181,12 +180,12 @@ def sync_customizations_for_doctype(data: dict, folder: str, filename: str = "") def scrub_dt_dn(dt: str, dn: str) -> tuple[str, str]: - """Returns in lowercase and code friendly names of doctype and name for certain types""" + """Return in lowercase and code friendly names of doctype and name for certain types.""" return scrub(dt), scrub(dn) def get_doc_path(module: str, doctype: str, name: str) -> str: - """Returns path of a doc in a module""" + """Return path of a doc in a module.""" return os.path.join(get_module_path(module), *scrub_dt_dn(doctype, name)) @@ -213,7 +212,7 @@ def export_doc(doctype, name, module=None): def get_doctype_module(doctype: str) -> str: - """Returns **Module Def** name of given doctype.""" + """Return **Module Def** name of given doctype.""" doctype_module_map = frappe.cache.get_value( "doctype_modules", generator=lambda: dict(frappe.qb.from_("DocType").select("name", "module").run()), @@ -226,7 +225,7 @@ def get_doctype_module(doctype: str) -> str: def load_doctype_module(doctype, module=None, prefix="", suffix=""): - """Returns the module object for given doctype. + """Return the module object for given doctype. Note: This will return the standard defined module object for the doctype irrespective of the `override_doctype_class` hook. diff --git a/frappe/patches/v11_0/replicate_old_user_permissions.py b/frappe/patches/v11_0/replicate_old_user_permissions.py index b66818d252..98ae220ff9 100644 --- a/frappe/patches/v11_0/replicate_old_user_permissions.py +++ b/frappe/patches/v11_0/replicate_old_user_permissions.py @@ -37,7 +37,7 @@ def execute(): def get_doctypes_to_skip(doctype, user): - """Returns doctypes to be skipped from user permission check""" + """Return doctypes to be skipped from user permission check.""" doctypes_to_skip = [] valid_perms = get_user_valid_perms(user) or [] for perm in valid_perms: diff --git a/frappe/patches/v13_0/queryreport_columns.py b/frappe/patches/v13_0/queryreport_columns.py index e9176952d4..61689c46de 100644 --- a/frappe/patches/v13_0/queryreport_columns.py +++ b/frappe/patches/v13_0/queryreport_columns.py @@ -7,7 +7,7 @@ import frappe def execute(): - """Convert Query Report json to support other content""" + """Convert Query Report json to support other content.""" records = frappe.get_all("Report", filters={"json": ["!=", ""]}, fields=["name", "json"]) for record in records: jstr = record["json"] diff --git a/frappe/permissions.py b/frappe/permissions.py index db1906dada..a334cc5722 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -63,8 +63,8 @@ def has_permission( *, parent_doctype=None, ): - """Returns True if user has permission `ptype` for given `doctype`. - If `doc` is passed, it also checks user, share and owner permissions. + """Return True if user has permission `ptype` for given `doctype`. + If `doc` is passed, also check user, share and owner permissions. :param doctype: DocType to check permission for :param ptype: Permission Type to check @@ -159,7 +159,7 @@ def has_permission( def get_doc_permissions(doc, user=None, ptype=None): - """Returns a dict of evaluated permissions for given `doc` like `{"read":1, "write":1}`""" + """Return a dict of evaluated permissions for given `doc` like `{"read":1, "write":1}`""" if not user: user = frappe.session.user @@ -204,7 +204,7 @@ def get_doc_permissions(doc, user=None, ptype=None): def get_role_permissions(doctype_meta, user=None, is_owner=None): """ - Returns dict of evaluated role permissions like + Return dict of evaluated role permissions like: { "read": 1, "write": 0, @@ -272,7 +272,7 @@ def get_user_permissions(user): def has_user_permission(doc, user=None): - """Returns True if User is allowed to view considering User Permissions""" + """Return True if User is allowed to view considering User Permissions.""" from frappe.core.doctype.user_permission.user_permission import get_user_permissions user_permissions = get_user_permissions(user) @@ -374,7 +374,7 @@ def has_user_permission(doc, user=None): def has_controller_permissions(doc, ptype, user=None): - """Returns controller permissions if defined. None if not defined""" + """Return controller permissions if defined, None if not defined.""" if not user: user = frappe.session.user @@ -415,7 +415,7 @@ def get_valid_perms(doctype=None, user=None): def get_all_perms(role): - """Returns valid permissions for a given role""" + """Return valid permissions for a given role.""" perms = frappe.get_all("DocPerm", fields="*", filters=dict(role=role)) custom_perms = frappe.get_all("Custom DocPerm", fields="*", filters=dict(role=role)) doctypes_with_custom_perms = frappe.get_all("Custom DocPerm", pluck="parent", distinct=True) @@ -462,7 +462,7 @@ def get_roles(user=None, with_standard=True): def get_doctype_roles(doctype, access_type="read"): - """Returns a list of roles that are allowed to access passed doctype.""" + """Return a list of roles that are allowed to access the given `doctype`.""" meta = frappe.get_meta(doctype) return [d.role for d in meta.get("permissions") if d.get(access_type)] @@ -474,7 +474,7 @@ def get_perms_for(roles, perm_doctype="DocPerm"): def get_doctypes_with_custom_docperms(): - """Returns all the doctypes with Custom Docperms""" + """Return all the doctypes with Custom Docperms.""" doctypes = frappe.get_all("Custom DocPerm", fields=["parent"], distinct=1) return [d.parent for d in doctypes] @@ -655,22 +655,18 @@ def get_doc_name(doc): def allow_everything(): - """ - returns a dict with access to everything - eg. {"read": 1, "write": 1, ...} - """ + """Return a dict with access to everything, eg. {"read": 1, "write": 1, ...}.""" return {ptype: 1 for ptype in rights} def get_allowed_docs_for_doctype(user_permissions, doctype): - """Returns all the docs from the passed user_permissions that are - allowed under provided doctype""" + """Return all the docs from the passed `user_permissions` that are allowed under provided doctype.""" return filter_allowed_docs_for_doctype(user_permissions, doctype, with_default_doc=False) def filter_allowed_docs_for_doctype(user_permissions, doctype, with_default_doc=True): - """Returns all the docs from the passed user_permissions that are - allowed under provided doctype along with default doc value if with_default_doc is set""" + """Return all the docs from the passed `user_permissions` that are + allowed under provided doctype along with default doc value if `with_default_doc` is set.""" allowed_doc = [] default_doc = None for doc in user_permissions: diff --git a/frappe/query_builder/custom.py b/frappe/query_builder/custom.py index 3203c72b8d..e209852891 100644 --- a/frappe/query_builder/custom.py +++ b/frappe/query_builder/custom.py @@ -88,11 +88,7 @@ class ConstantColumn(Term): alias = None def __init__(self, value: str) -> None: - """[ Returns a pseudo column with a constant value in all the rows] - - Args: - value (str): [ Value of the column ] - """ + """Return a pseudo column with the given constant `value` in all the rows.""" self.value = value def get_sql(self, quote_char: str | None = None, **kwargs: Any) -> str: diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index c1af0d9aa2..c33f890a39 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -15,24 +15,20 @@ class NamedParameterWrapper: self.parameters = {} def get_sql(self, param_value: Any, **kwargs) -> str: - """returns SQL for a parameter, while adding the real value in a dict + """Return SQL for a parameter, while adding the real value in a dict. Args: - param_value (Any): Value of the parameter + param_value (Any): Value of the parameter - Returns: - str: parameter used in the SQL query + Return: + str: parameter used in the SQL query """ param_key = f"%(param{len(self.parameters) + 1})s" self.parameters[param_key[2:-2]] = param_value return param_key def get_parameters(self) -> dict[str, Any]: - """get dict with parameters and values - - Returns: - Dict[str, Any]: parameter dict - """ + """Get dict with parameters and values.""" return self.parameters diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index f4ab85ac6b..93dde3d2b8 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -42,13 +42,10 @@ class BuilderIdentificationFailed(Exception): def get_query_builder(type_of_db: str) -> Postgres | MariaDB: - """[return the query builder object] + """Return the query builder object. Args: - type_of_db (str): [string value of the db used] - - Returns: - Query: [Query object] + type_of_db: string value of the db used """ db = db_type_is(type_of_db) picks = {db_type_is.MARIADB: MariaDB, db_type_is.POSTGRES: Postgres} diff --git a/frappe/rate_limiter.py b/frappe/rate_limiter.py index f1451ef91c..332a5a8070 100644 --- a/frappe/rate_limiter.py +++ b/frappe/rate_limiter.py @@ -110,7 +110,7 @@ def rate_limit( :param ip_based: flag to allow ip based rate-limiting :type ip_based: Boolean - :returns: a decorator function that limit the number of requests per endpoint + Return: a decorator function that limit the number of requests per endpoint """ def ratelimit_decorator(fn): diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py index 151799a6e1..3efa8b4159 100644 --- a/frappe/search/full_text_search.py +++ b/frappe/search/full_text_search.py @@ -107,16 +107,13 @@ class FullTextSearch: writer.commit(optimize=True) - def search(self, text, scope=None, limit=20): - """Search from the current index + def search(self, text: str, scope: str | None = None, limit: int = 20) -> list[frappe._dict]: + """Search from the current index. Args: - text (str): String to search for - scope (str, optional): Scope to limit the search. Defaults to None. - limit (int, optional): Limit number of search results. Defaults to 20. - - Returns: - [List(_dict)]: Search results + text: String to search for + scope: Scope to limit the search. Defaults to None. + limit: Limit number of search results. Defaults to 20. """ ix = self.get_index() diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 2b35b86de7..5eabd01737 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -27,10 +27,9 @@ class WebsiteSearch(FullTextSearch): return "path" def get_items_to_index(self): - """Get all routes to be indexed, this includes the static pages - in www/ and routes from published documents + """Get all routes to be indexed, this includes the static pages in www/ and routes from published documents. - Returns: + Return: self (object): FullTextSearch Instance """ @@ -49,14 +48,13 @@ class WebsiteSearch(FullTextSearch): return self.get_items_to_index() - def get_document_to_index(self, route): - """Render a page and parse it using BeautifulSoup + def get_document_to_index(self, route: str) -> frappe._dict | None: + """Render a page and parse it using `BeautifulSoup`. Args: - path (str): route of the page to be parsed + path: route of the page to be parsed - Returns: - document (_dict): A dictionary with title, path and content + Return a dictionary with title, path and content. """ frappe.set_user("Guest") frappe.local.no_cache = True diff --git a/frappe/sessions.py b/frappe/sessions.py index b0eb7e7353..ef1e7f54f1 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -48,7 +48,7 @@ def clear_sessions(user=None, keep_current=False, force=False): def get_sessions_to_clear(user=None, keep_current=False): - """Returns sessions of the current user. Called at login / logout + """Return sessions of the current user. Called at login / logout. :param user: user name (default: current user) :param keep_current: keep current session (default: false) @@ -109,7 +109,7 @@ def clear_all_sessions(reason=None): def get_expired_sessions(): - """Returns list of expired sessions""" + """Return list of expired sessions.""" sessions = frappe.qb.DocType("Sessions") return ( diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 3cff14afa9..e6af22dd0c 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -43,14 +43,7 @@ CLI_CONTEXT = frappe._dict(sites=[TEST_SITE]) def clean(value) -> str: - """Strips and converts bytes to str - - Args: - value ([type]): [description] - - Returns: - [type]: [description] - """ + """Strip and convert bytes to str.""" if isinstance(value, bytes): value = value.decode() if isinstance(value, str): @@ -59,13 +52,13 @@ def clean(value) -> str: def missing_in_backup(doctypes: list, file: os.PathLike) -> list: - """Returns list of missing doctypes in the backup. + """Return list of missing doctypes in the backup. Args: doctypes (list): List of DocTypes to be checked file (str): Path of the database file - Returns: + Return: doctypes(list): doctypes that are missing in backup """ predicate = 'COPY public."tab{}"' if frappe.conf.db_type == "postgres" else "CREATE TABLE `tab{}`" @@ -76,14 +69,13 @@ def missing_in_backup(doctypes: list, file: os.PathLike) -> list: def exists_in_backup(doctypes: list, file: os.PathLike) -> bool: - """Checks if the list of doctypes exist in the database.sql.gz file supplied + """Check if the list of doctypes exist in the database.sql.gz file supplied. Args: doctypes (list): List of DocTypes to be checked file (str): Path of the database file - Returns: - bool: True if all tables exist + Return True if all tables exist. """ missing_doctypes = missing_in_backup(doctypes, file) return len(missing_doctypes) == 0 diff --git a/frappe/translate.py b/frappe/translate.py index 0f023c1f91..fbbe84a8ce 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -152,7 +152,7 @@ def set_default_language(lang): def get_lang_dict(): - """Returns all languages in dict format, full name is the key e.g. `{"english":"en"}`""" + """Return all languages in dict format, full name is the key e.g. `{"english":"en"}`.""" return dict( frappe.get_all("Language", fields=["language_name", "name"], order_by="modified", as_list=True) ) @@ -267,7 +267,7 @@ def clear_cache(): def get_messages_for_app(app, deduplicate=True): - """Returns all messages (list) for a specified `app`""" + """Return all messages (list) for a specified `app`.""" messages = [] modules = [frappe.unscrub(m) for m in frappe.local.app_modules[app]] @@ -479,12 +479,12 @@ def get_messages_from_custom_fields(app_name): def get_messages_from_page(name): - """Returns all translatable strings from a :class:`frappe.core.doctype.Page`""" + """Return all translatable strings from a :class:`frappe.core.doctype.Page`.""" return _get_messages_from_page_or_report("Page", name) def get_messages_from_report(name): - """Returns all translatable strings from a :class:`frappe.core.doctype.Report`""" + """Return all translatable strings from a :class:`frappe.core.doctype.Report`.""" report = frappe.get_doc("Report", name) messages = _get_messages_from_page_or_report( "Report", name, frappe.db.get_value("DocType", report.ref_doctype, "module") @@ -550,7 +550,7 @@ def get_server_messages(app): def get_messages_from_include_files(app_name=None): - """Returns messages from js files included at time of boot like desk.min.js for desk and web""" + """Return messages from js files included at time of boot like desk.min.js for desk and web.""" from frappe.utils.jinja_globals import bundled_asset messages = [] @@ -584,7 +584,7 @@ def get_all_messages_from_js_files(app_name=None): def get_messages_from_file(path: str) -> list[tuple[str, str, str | None, int]]: - """Returns a list of transatable strings from a code file + """Return a list of transatable strings from a code file. :param path: path of the code file """ @@ -913,7 +913,7 @@ def write_csv_file(path, app_messages, lang_dict): def get_untranslated(lang, untranslated_file, get_all=False, app="_ALL_APPS"): - """Returns all untranslated strings for a language and writes in a file + """Return all untranslated strings for a language and write in a file. :param lang: Language code. :param untranslated_file: Output file path. @@ -1188,7 +1188,7 @@ def get_translator_url(): @frappe.whitelist(allow_guest=True) def get_all_languages(with_language_name: bool = False) -> list: - """Returns all enabled language codes ar, ch etc""" + """Return all enabled language codes ar, ch etc.""" def get_language_codes(): return frappe.get_all("Language", filters={"enabled": 1}, pluck="name") diff --git a/frappe/twofactor.py b/frappe/twofactor.py index c166af56ce..6b23e3c0e4 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -42,7 +42,7 @@ def toggle_two_factor_auth(state, roles=None): def two_factor_is_enabled(user=None): - """Returns True if 2FA is enabled.""" + """Return True if 2FA is enabled.""" enabled = cint(frappe.db.get_single_value("System Settings", "enable_two_factor_auth")) if enabled: bypass_two_factor_auth = cint( diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index ee52d280e1..3e661978a6 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -123,7 +123,7 @@ def validate_phone_number_with_country_code(phone_number: str, fieldname: str) - def validate_phone_number(phone_number, throw=False): - """Returns True if valid phone number""" + """Return True if valid phone number.""" if not phone_number: return False @@ -139,9 +139,10 @@ def validate_phone_number(phone_number, throw=False): def validate_name(name, throw=False): - """Returns True if the name is valid - valid names may have unicode and ascii characters, dash, quotes, numbers - anything else is considered invalid + """Return True if the name is valid + + * valid names may have unicode and ascii characters, dash, quotes, numbers + * anything else is considered invalid Note: "Name" here is name of a person, not the primary key in Frappe doctypes. """ @@ -219,14 +220,11 @@ def validate_url( valid_schemes: str | Container[str] | None = None, ) -> bool: """ - Checks whether `txt` has a valid URL string + Return True if `txt` represents a valid URL. - Parameters: - throw (`bool`): throws a validationError if URL is not valid - valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this - - Returns: - bool: if `txt` represents a valid URL + Args: + throw: throws a validationError if URL is not valid + valid_schemes: if provided checks the given URL's scheme against this """ url = urlparse(txt) is_valid = bool(url.netloc) @@ -252,7 +250,7 @@ def random_string(length: int) -> str: def has_gravatar(email: str) -> str: - """Returns gravatar url if user has set an avatar at gravatar.com""" + """Return gravatar url if user has set an avatar at gravatar.com.""" import requests if frappe.flags.in_import or frappe.flags.in_install or frappe.flags.in_test: @@ -285,9 +283,7 @@ def get_gravatar(email: str) -> str: def get_traceback(with_context=False) -> str: - """ - Returns the traceback of the Exception - """ + """Return the traceback of the Exception.""" from traceback_with_variables import iter_exc_lines exc_type, exc_value, exc_tb = sys.exc_info() @@ -350,9 +346,7 @@ def log(event, details): def dict_to_str(args: dict[str, Any], sep: str = "&") -> str: - """ - Converts a dictionary to URL - """ + """Convert a dictionary to URL.""" return sep.join(f"{str(k)}=" + quote(str(args[k] or "")) for k in list(args)) @@ -383,9 +377,7 @@ def set_default(key, val): def remove_blanks(d: dict) -> dict: - """ - Returns d with empty ('' or None) values stripped. Mutates inplace. - """ + """Return d with empty ('' or None) values stripped. Mutates inplace.""" for k, v in tuple(d.items()): if v == "" or v == None: del d[k] @@ -393,14 +385,12 @@ def remove_blanks(d: dict) -> dict: def strip_html_tags(text): - """Remove html tags from text""" + """Remove html tags from the given `text`.""" return HTML_TAGS_PATTERN.sub("", text) def get_file_timestamp(fn): - """ - Returns timestamp of the given file - """ + """Return timestamp of the given file.""" from frappe.utils import cint try: @@ -566,7 +556,7 @@ def touch_file(path): def get_test_client(use_cookies=True) -> Client: - """Returns an test instance of the Frappe WSGI""" + """Return an test instance of the Frappe WSGI.""" from frappe.app import application return Client(application, use_cookies=use_cookies) @@ -590,7 +580,7 @@ def call_hook_method(hook, *args, **kwargs): def is_cli() -> bool: - """Returns True if current instance is being run via a terminal""" + """Return True if current instance is being run via a terminal.""" invoked_from_terminal = False try: invoked_from_terminal = bool(os.get_terminal_size()) @@ -835,11 +825,12 @@ def parse_json(val): def get_db_count(*args): """ - Pass a doctype or a series of doctypes to get the count of docs in them + Pass a doctype or a series of doctypes to get the count of docs in them. + Parameters: *args: Variable length argument list of doctype names whose doc count you need - Returns: + Return: dict: A dict with the count values. Example: @@ -859,7 +850,7 @@ def call(fn, *args, **kwargs): Parameters: fn: frappe function to be called - Returns: + Return: based on the function you call: output of the function you call Example: @@ -912,7 +903,7 @@ def get_safe_filters(filters): def create_batch(iterable: Iterable, size: int) -> Generator[Iterable, None, None]: - """Convert an iterable to multiple batches of constant size of batch_size + """Convert an iterable to multiple batches of constant size of batch_size. Args: iterable (Iterable): Iterable object which is subscriptable @@ -992,12 +983,12 @@ def get_assets_json(): def get_bench_relative_path(file_path): - """Fixes paths relative to the bench root directory if exists and returns the absolute path + """Fix paths relative to the bench root directory if exists and return the absolute path. Args: file_path (str, Path): Path of a file that exists on the file system - Returns: + Return: str: Absolute path of the file_path """ if not os.path.exists(file_path): diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 2c09604916..d89003a81d 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -394,7 +394,7 @@ def get_queue_list(queue_list=None, build_queue_name=False): def get_workers(queue=None): - """Returns a list of Worker objects tied to a queue object if queue is passed, else returns a list of all workers""" + """Return a list of Worker objects tied to a queue object if queue is passed, else return a list of all workers.""" if queue: return Worker.all(queue=queue) else: @@ -402,7 +402,7 @@ def get_workers(queue=None): def get_running_jobs_in_queue(queue): - """Returns a list of Jobs objects that are tied to a queue object and are currently running""" + """Return a list of Jobs objects that are tied to a queue object and are currently running.""" jobs = [] workers = get_workers(queue) for worker in workers: @@ -413,7 +413,7 @@ def get_running_jobs_in_queue(queue): def get_queue(qtype, is_async=True): - """Returns a Queue object tied to a redis connection""" + """Return a Queue object tied to a redis connection.""" validate_queue(qtype) return Queue(generate_qname(qtype), connection=get_redis_conn(), is_async=is_async) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 194b477e1a..e716ff6e7a 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -475,11 +475,12 @@ download only after 24 hours.""".format( @frappe.whitelist() -def fetch_latest_backups(partial=False): - """Fetches paths of the latest backup taken in the last 30 days - Only for: System Managers +def fetch_latest_backups(partial=False) -> dict: + """Fetch paths of the latest backup taken in the last 30 days. - Returns: + Note: Only for System Managers + + Return: dict: relative Backup Paths """ frappe.only_for("System Manager") @@ -583,13 +584,8 @@ def delete_temp_backups(older_than=24): os.remove(this_file_path) -def is_file_old(file_path, older_than=24): - """ - Checks if file exists and is older than specified hours - Returns -> - True: file does not exist or file is old - False: file is new - """ +def is_file_old(file_path, older_than=24) -> bool: + """Return True if file exists and is older than specified hours.""" if os.path.isfile(file_path): from datetime import timedelta diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index 3da9c5d757..26b74473a3 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -131,7 +131,7 @@ def get_versions(): def get_app_branch(app): - """Returns branch of an app""" + """Return branch of an app.""" try: with open(os.devnull, "wb") as null_stream: result = subprocess.check_output( @@ -198,15 +198,14 @@ def check_for_update(): add_message_to_redis(updates) -def parse_latest_non_beta_release(response): - """ - Parses the response JSON for all the releases and returns the latest non prerelease +def parse_latest_non_beta_release(response: list) -> list | None: + """Parse the response JSON for all the releases and return the latest non prerelease. + + Args: - Parameters response (list): response object returned by github - Returns - json : json object pertaining to the latest non-beta release + Return a json object pertaining to the latest non-beta release """ version_list = [ release.get("tag_name").strip("v") for release in response if not release.get("prerelease") @@ -218,14 +217,13 @@ def parse_latest_non_beta_release(response): return None -def check_release_on_github(app: str): - """ - Check the latest release for a given Frappe application hosted on Github. +def check_release_on_github(app: str) -> tuple | None: + """Check the latest release for a given Frappe application hosted on Github. Args: - app (str): The name of the Frappe application. + app: The name of the Frappe application. - Returns: + Return: tuple(Version, str): The semantic version object of the latest release and the organization name, if the application exists, otherwise None. """ diff --git a/frappe/utils/data.py b/frappe/utils/data.py index d8a8ae4b01..dd92d72318 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -90,7 +90,7 @@ def get_start_of_week_index() -> int: def is_invalid_date_string(date_string: str) -> bool: - """Returns True if the date string is invalid or None or empty.""" + """Return True if the date string is invalid or None or empty.""" # dateutil parser does not agree with dates like "0001-01-01" or "0000-00-00" return not isinstance(date_string, str) or ( (not date_string) or (date_string or "").startswith(("0001-01-01", "0000-00-00")) @@ -101,7 +101,7 @@ def getdate( string_date: Optional["DateTimeLikeObject"] = None, parse_day_first: bool = False ) -> datetime.date | None: """ - Converts string date (yyyy-mm-dd) to datetime.date object. + Convert string date (yyyy-mm-dd) to datetime.date object. If no input is provided, current date is returned. """ if not string_date: @@ -126,7 +126,7 @@ def getdate( def get_datetime( datetime_str: Optional["DateTimeLikeObject"] = None, ) -> datetime.datetime | None: - """Returns the below mentioned values based on the given `datetime_str`: + """Return the below mentioned values based on the given `datetime_str`: * If `datetime_str` is None, returns datetime object of current datetime * If `datetime_str` is already a datetime object, returns the same @@ -159,8 +159,9 @@ def get_datetime( def get_timedelta(time: str | None = None) -> datetime.timedelta | None: - """Return `datetime.timedelta` object from string value of a - valid time format. Returns None if `time` is not a valid format + """Return `datetime.timedelta` object from string value of a valid time format. + + Return None if `time` is not a valid format. Args: time (str): A valid time representation. This string is parsed @@ -168,7 +169,7 @@ def get_timedelta(time: str | None = None) -> datetime.timedelta | None: '0:0:0', '17:21:00', '2012-01-19 17:21:00'. Checkout https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse - Returns: + Return: datetime.timedelta: Timedelta object equivalent of the passed `time` string """ time = time or "0:0:0" @@ -188,7 +189,7 @@ def get_timedelta(time: str | None = None) -> datetime.timedelta | None: def to_timedelta(time_str: str | datetime.time) -> datetime.timedelta: - """Returns a `datetime.timedelta` object from the given string or `datetime.time` object. + """Return a `datetime.timedelta` object from the given string or `datetime.time` object. If the given argument is not a string or a `datetime.time` object, it is returned as is. """ if isinstance(time_str, datetime.time): @@ -295,32 +296,32 @@ def add_to_date( def add_days(date: DateTimeLikeObject, days: NumericType) -> DateTimeLikeObject: - """Returns a new date after adding the given number of `days` to the given `date`.""" + """Return a new date after adding the given number of `days` to the given `date`.""" return add_to_date(date, days=days) def add_months(date: DateTimeLikeObject, months: NumericType) -> DateTimeLikeObject: - """Returns a new date after adding the given number of `months` to the given `date`.""" + """Return a new date after adding the given number of `months` to the given `date`.""" return add_to_date(date, months=months) def add_years(date: DateTimeLikeObject, years: NumericType) -> DateTimeLikeObject: - """Returns a new date after adding the given number of `years` to the given `date`.""" + """Return a new date after adding the given number of `years` to the given `date`.""" return add_to_date(date, years=years) def date_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject) -> int: - """Returns the difference between given two dates in days.""" + """Return the difference between given two dates in days.""" return days_diff(string_ed_date, string_st_date) def days_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject) -> int: - """Returns the difference between given two dates in days.""" + """Return the difference between given two dates in days.""" return (getdate(string_ed_date) - getdate(string_st_date)).days def month_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject) -> int: - """Returns the difference between given two dates in months.""" + """Return the difference between given two dates in months.""" ed_date = getdate(string_ed_date) st_date = getdate(string_st_date) return (ed_date.year - st_date.year) * 12 + ed_date.month - st_date.month + 1 @@ -329,32 +330,32 @@ def month_diff(string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeO def time_diff( string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject ) -> datetime.timedelta: - """Returns the difference between given two dates as `datetime.timedelta` object.""" + """Return the difference between given two dates as `datetime.timedelta` object.""" return get_datetime(string_ed_date) - get_datetime(string_st_date) def time_diff_in_seconds( string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject ) -> float: - """Returns the difference between given two dates in seconds.""" + """Return the difference between given two dates in seconds.""" return time_diff(string_ed_date, string_st_date).total_seconds() def time_diff_in_hours( string_ed_date: DateTimeLikeObject, string_st_date: DateTimeLikeObject ) -> float: - """Returns the difference between given two dates in hours.""" + """Return the difference between given two dates in hours.""" return round(float(time_diff(string_ed_date, string_st_date).total_seconds()) / 3600, 6) def now_datetime() -> datetime.datetime: - """Returns the current datetime in system timezone.""" + """Return the current datetime in system timezone.""" dt = convert_utc_to_system_timezone(datetime.datetime.now(pytz.UTC)) return dt.replace(tzinfo=None) def get_timestamp(date: Optional["DateTimeLikeObject"] = None) -> float: - """Returns the Unix timestamp (seconds since Epoch) for the given `date`. + """Return the Unix timestamp (seconds since Epoch) for the given `date`. If `date` is None, the current timestamp is returned. """ return time.mktime(getdate(date).timetuple()) @@ -370,7 +371,7 @@ def _get_system_timezone(): def get_system_timezone() -> str: - """Returns the system timezone.""" + """Return the system timezone.""" if frappe.local.flags.in_test: return _get_system_timezone() @@ -389,19 +390,19 @@ def convert_utc_to_timezone(utc_timestamp: datetime.datetime, time_zone: str) -> def get_datetime_in_timezone(time_zone: str) -> datetime.datetime: - """Returns the current datetime in the given timezone (e.g. 'Asia/Kolkata').""" + """Return the current datetime in the given timezone (e.g. 'Asia/Kolkata').""" utc_timestamp = datetime.datetime.now(pytz.UTC) return convert_utc_to_timezone(utc_timestamp, time_zone) def convert_utc_to_system_timezone(utc_timestamp: datetime.datetime) -> datetime.datetime: - """Returns the given UTC `datetime` timestamp in system timezone.""" + """Return the given UTC `datetime` timestamp in system timezone.""" time_zone = get_system_timezone() return convert_utc_to_timezone(utc_timestamp, time_zone) def now() -> str: - """return current datetime as yyyy-mm-dd hh:mm:ss""" + """Return current datetime as `yyyy-mm-dd hh:mm:ss`.""" if frappe.flags.current_date: return ( getdate(frappe.flags.current_date).strftime(DATE_FORMAT) @@ -413,17 +414,17 @@ def now() -> str: def nowdate() -> str: - """return current date as yyyy-mm-dd""" + """Return current date as `yyyy-mm-dd`.""" return now_datetime().strftime(DATE_FORMAT) def today() -> str: - """Returns today's date in `yyyy-mm-dd` format.""" + """Return today's date in `yyyy-mm-dd` format.""" return nowdate() def get_abbr(string: str, max_len: int = 2) -> str: - """Returns the abbreviation of the given string. + """Return the abbreviation of the given string. Examples: @@ -431,7 +432,7 @@ def get_abbr(string: str, max_len: int = 2) -> str: * "Jenny Jane Doe" => "JJ" (default, `max_len` = 2) * "Jenny Jane Doe" => "JJD" (`max_len` = 3) - Returns "?" if the given string is empty. + Return "?" if the given string is empty. """ abbr = "" for part in string.split(" "): @@ -442,7 +443,7 @@ def get_abbr(string: str, max_len: int = 2) -> str: def nowtime() -> str: - """Returns current time (system timezone) in `hh:mm:ss` format.""" + """Return current time (system timezone) in `hh:mm:ss` format.""" return now_datetime().strftime(TIME_FORMAT) @@ -460,9 +461,9 @@ def get_first_day(dt, d_years=0, d_months=0, as_str: Literal[True] = False) -> s def get_first_day( dt, d_years: int = 0, d_months: int = 0, as_str: bool = False ) -> str | datetime.date: - """ - Returns the first day of the month for the date specified by date object - Also adds `d_years` and `d_months` if specified + """Return the first day of the month for the date specified by date object. + + Also, add `d_years` and `d_months` if specified. """ dt = getdate(dt) @@ -492,7 +493,7 @@ def get_quarter_start(dt: DateTimeLikeObject | None = None, as_str: Literal[True def get_quarter_start( dt: DateTimeLikeObject | None = None, as_str: bool = False ) -> str | datetime.date: - """Returns the start date of the quarter for the given datetime like object (`dt`). + """Return the start date of the quarter for the given datetime like object (`dt`). If `dt` is None, the current quarter start date is returned. """ @@ -513,7 +514,7 @@ def get_first_day_of_week(dt: DateTimeLikeObject, as_str: Literal[True] = False) def get_first_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date | str: - """Returns the first day of the week (as per System Settings or Sunday by default) for the given datetime like object (`dt`). + """Return the first day of the week (as per System Settings or Sunday by default) for the given datetime like object (`dt`). If `as_str` is True, the first day of the week is returned as a string in `yyyy-mm-dd` format. """ @@ -548,7 +549,7 @@ def get_year_start(dt: DateTimeLikeObject, as_str: Literal[True] = False) -> str def get_year_start(dt: DateTimeLikeObject, as_str=False) -> str | datetime.date: - """Returns the start date of the year for the given date (`dt`).""" + """Return the start date of the year for the given date (`dt`).""" dt = getdate(dt) date = datetime.date(dt.year, 1, 1) return date.strftime(DATE_FORMAT) if as_str else date @@ -565,7 +566,7 @@ def get_last_day_of_week(dt: DateTimeLikeObject, as_str: Literal[True] = False) def get_last_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date | str: - """Returns the last day of the week (first day is taken from System Settings or Sunday by default) for the given datetime like object (`dt`). + """Return the last day of the week (first day is taken from System Settings or Sunday by default) for the given datetime like object (`dt`). If `as_str` is True, the last day of the week is returned as a string in `yyyy-mm-dd` format. """ @@ -575,8 +576,8 @@ def get_last_day_of_week(dt: DateTimeLikeObject, as_str=False) -> datetime.date def get_last_day(dt): - """ - Returns last day of the month using: + """Return last day of the month using: + `get_first_day(dt, 0, 1) + datetime.timedelta(-1)` """ return get_first_day(dt, 0, 1) + datetime.timedelta(-1) @@ -603,7 +604,7 @@ def get_quarter_ending(dt: DateTimeLikeObject | None = None, as_str: Literal[Tru def get_quarter_ending( date: DateTimeLikeObject | None = None, as_str=False ) -> str | datetime.date: - """Returns the end date of the quarter for the given datetime like object (`date`). + """Return the end date of the quarter for the given datetime like object (`date`). If `date` is None, the current quarter end date is returned. If `as_str` is True, the end date of the quarter is returned as a string in `yyyy-mm-dd` format. @@ -635,7 +636,7 @@ def get_year_ending(dt: DateTimeLikeObject | None = None, as_str: Literal[True] def get_year_ending(date: DateTimeLikeObject | None = None, as_str=False) -> datetime.date | str: - """Returns the end date of the year for the given datetime like object (`date`). + """Return the end date of the year for the given datetime like object (`date`). If `date` is None, the current year end date is returned. If `as_str` is True, the end date of the year is returned as a string in `yyyy-mm-dd` format. @@ -649,7 +650,7 @@ def get_year_ending(date: DateTimeLikeObject | None = None, as_str=False) -> dat def get_time( time_str: str | datetime.datetime | datetime.time | datetime.timedelta, ) -> datetime.time: - """Returns a `datetime.time` object for the given `time_str`. + """Return a `datetime.time` object for the given `time_str`. If the given argument is already a `datetime.time` object, it is returned as is.""" @@ -668,21 +669,21 @@ def get_time( def get_datetime_str(datetime_obj: DateTimeLikeObject) -> str: - """Returns the given datetime like object (datetime.date, datetime.datetime, string) as a string in `yyyy-mm-dd hh:mm:ss` format.""" + """Return the given datetime like object (datetime.date, datetime.datetime, string) as a string in `yyyy-mm-dd hh:mm:ss` format.""" if isinstance(datetime_obj, str): datetime_obj = get_datetime(datetime_obj) return datetime_obj.strftime(DATETIME_FORMAT) def get_date_str(date_obj: DateTimeLikeObject) -> str: - """Returns the given datetime like object (datetime.date, datetime.datetime, string) as a string in `yyyy-mm-dd` format.""" + """Return the given datetime like object (datetime.date, datetime.datetime, string) as a string in `yyyy-mm-dd` format.""" if isinstance(date_obj, str): date_obj = get_datetime(date_obj) return date_obj.strftime(DATE_FORMAT) def get_time_str(timedelta_obj: datetime.timedelta | str) -> str: - """Returns the given timedelta object as a string in `hh:mm:ss` format.""" + """Return the given timedelta object as a string in `hh:mm:ss` format.""" if isinstance(timedelta_obj, str): timedelta_obj = to_timedelta(timedelta_obj) @@ -713,7 +714,8 @@ def get_user_time_format() -> str: def format_date( string_date=None, format_string: str | None = None, parse_day_first: bool = False ) -> str: - """Converts the given string date to :data:`user_date_format` + """Convert the given string date to :data:`user_date_format`. + User format specified in defaults Examples: @@ -746,7 +748,8 @@ formatdate = format_date # For backwards compatibility def format_time(time_string=None, format_string: str | None = None) -> str: - """Converts the given string time to :data:`user_time_format` + """Convert the given string time to :data:`user_time_format`. + User format specified in defaults Examples: @@ -773,7 +776,7 @@ def format_time(time_string=None, format_string: str | None = None) -> str: def format_datetime(datetime_string: DateTimeLikeObject, format_string: str | None = None) -> str: - """Converts the given string time to :data:`user_datetime_format` + """Convert the given string time to :data:`user_datetime_format` User format specified in defaults Examples: @@ -801,9 +804,9 @@ def format_datetime(datetime_string: DateTimeLikeObject, format_string: str | No def format_duration(seconds, hide_days=False): - """Converts the given duration value in float(seconds) to duration format + """Convert the given duration value in float(seconds) to duration format. - example: converts 12885 to '3h 34m 45s' where 12885 = seconds in float + example: convert 12885 to '3h 34m 45s' where 12885 = seconds in float """ seconds = cint(seconds) @@ -837,9 +840,9 @@ def format_duration(seconds, hide_days=False): def duration_to_seconds(duration): - """Converts the given duration formatted value to duration value in seconds + """Convert the given duration formatted value to duration value in seconds. - example: converts '3h 34m 45s' to 12885 (value in seconds) + example: convert '3h 34m 45s' to 12885 (value in seconds) """ validate_duration_format(duration) value = 0 @@ -876,15 +879,15 @@ def validate_duration_format(duration): def get_weekdays() -> list[str]: - """Returns a list of weekday names. + """Return a list of weekday names. - Return value: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] + Return value: ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] """ return ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] def get_weekday(datetime: DateTimeLikeObject | None = None) -> str: - """Returns the weekday name (e.g. 'Sunday') for the given datetime like object (datetime.date, datetime.datetime, string). + """Return the weekday name (e.g. 'Sunday') for the given datetime like object (datetime.date, datetime.datetime, string). If `datetime` argument is not provided, the current weekday name is returned. """ @@ -901,7 +904,7 @@ def get_weekday(datetime: DateTimeLikeObject | None = None) -> str: def get_timespan_date_range( timespan: TimespanOptions, ) -> tuple[datetime.datetime, datetime.datetime] | None: - """Returns the date range (start_date, end_date) tuple for the given timespan.""" + """Return the date range (start_date, end_date) tuple for the given timespan.""" today = getdate() match timespan: @@ -974,8 +977,8 @@ def get_timespan_date_range( return -def global_date_format(date, format="long"): - """returns localized date in the form of January 1, 2012""" +def global_date_format(date: DateTimeLikeObject, format="long") -> str: + """Return localized date in the form of 'January 1, 2012'.""" import babel.dates date = getdate(date) @@ -985,7 +988,7 @@ def global_date_format(date, format="long"): def has_common(l1: typing.Hashable, l2: typing.Hashable) -> bool: - """Returns truthy value if there are common elements in lists l1 and l2""" + """Return truthy value if there are common elements in lists l1 and l2.""" return set(l1) & set(l2) @@ -1089,13 +1092,13 @@ def flt(s: NumericType | str, precision: int | None = None) -> float: def flt( s: NumericType | str, precision: int | None = None, rounding_method: str | None = None ) -> float: - """Convert to float (ignoring commas in string) + """Convert to float (ignoring commas in string). :param s: Number in string or other numeric format. :param precision: optional argument to specify precision for rounding. :returns: Converted number in python float type. - Returns 0 if input can not be converted to float. + Return 0 if input can not be converted to float. Examples: @@ -1124,12 +1127,12 @@ def flt( def cint(s: NumericType | str, default: int = 0) -> int: - """Convert to integer + """Convert to integer. :param s: Number in string or other numeric format. :returns: Converted number in python integer type. - Returns default if input can not be converted to integer. + Return default if input cannot be converted to integer. Examples: >>> cint("100") @@ -1144,21 +1147,8 @@ def cint(s: NumericType | str, default: int = 0) -> int: return default -def floor(s): - """ - A number representing the largest integer less than or equal to the specified number - - Parameters - ---------- - s : int or str or Decimal object - The mathematical value to be floored - - Returns - ------- - int - number representing the largest integer less than or equal to the specified number - - """ +def floor(s: NumericType | str) -> int: + """Return a number representing the largest integer less than or equal to the specified number.""" try: num = cint(math.floor(flt(s))) except Exception: @@ -1166,21 +1156,8 @@ def floor(s): return num -def ceil(s): - """ - The smallest integer greater than or equal to the given number - - Parameters - ---------- - s : int or str or Decimal object - The mathematical value to be ceiled - - Returns - ------- - int - smallest integer greater than or equal to the given number - - """ +def ceil(s: NumericType | str) -> int: + """Return the smallest integer greater than or equal to the given number.""" try: num = cint(math.ceil(flt(s))) except Exception: @@ -1189,12 +1166,13 @@ def ceil(s): def cstr(s, encoding="utf-8") -> str: - """Converts the given argument to string.""" + """Convert the given argument to string.""" return frappe.as_unicode(s, encoding) def sbool(x: str) -> bool | Any: - """Converts str object to Boolean if possible. + """Convert str object to Boolean if possible. + Example: "true" becomes True "1" becomes True @@ -1203,8 +1181,7 @@ def sbool(x: str) -> bool | Any: Args: x (str): String to be converted to Bool - Returns: - object: Returns Boolean or x + Return Boolean or x. """ try: val = x.lower() @@ -1361,7 +1338,7 @@ def encode(obj, encoding="utf-8"): def parse_val(v): - """Converts to simple datatypes from SQL query results""" + """Convert to simple datatypes from SQL query results.""" if isinstance(v, (datetime.date, datetime.datetime)): v = str(v) elif isinstance(v, datetime.timedelta): @@ -1377,9 +1354,7 @@ def fmt_money( currency: str | None = None, format: str | None = None, ) -> str: - """ - Convert to string with commas for thousands, millions etc - """ + """Convert to string with commas for thousands, millions etc.""" number_format = format or frappe.db.get_default("number_format") or "#,###.##" if precision is None: precision = cint(frappe.db.get_default("currency_precision")) or None @@ -1485,9 +1460,7 @@ def money_in_words( main_currency: str | None = None, fraction_currency: str | None = None, ): - """ - Returns string in words with currency and fraction currency. - """ + """Return string in words with currency and fraction currency.""" from frappe.utils import get_defaults _ = frappe._ @@ -1557,9 +1530,7 @@ def money_in_words( # convert number to words # def in_words(integer: int, in_million=True) -> str: - """ - Returns string in words for the given integer. - """ + """Return string in words for the given integer.""" from num2words import num2words locale = "en_IN" if not in_million else frappe.local.lang @@ -1574,14 +1545,14 @@ def in_words(integer: int, in_million=True) -> str: def is_html(text: str) -> bool: - """Returns True if the given `text` contains any HTML tags.""" + """Return True if the given `text` contains any HTML tags.""" if not isinstance(text, str): return False return HTML_TAG_PATTERN.search(text) def is_image(filepath: str) -> bool: - """Returns True if the given `filepath` points to an image file.""" + """Return True if the given `filepath` points to an image file.""" from mimetypes import guess_type # filepath can be https://example.com/bed.jpg?v=129 @@ -1699,7 +1670,7 @@ def pretty_date(iso_datetime: datetime.datetime | str) -> str: def comma_or(some_list: list | tuple, add_quotes=True) -> str: - """Returns the given list or tuple as a comma separated string with the last item joined by 'or'. + """Return the given list or tuple as a comma separated string with the last item joined by 'or'. e.g. ['a', 'b', 'c'] -> 'a, b or c' If `add_quotes` is True, each item in the list will be wrapped in single quotes. @@ -1709,7 +1680,7 @@ def comma_or(some_list: list | tuple, add_quotes=True) -> str: def comma_and(some_list: list | tuple, add_quotes=True) -> str: - """Returns the given list or tuple as a comma separated string with the last item joined by 'and'. + """Return the given list or tuple as a comma separated string with the last item joined by 'and'. e.g. ['a', 'b', 'c'] -> 'a, b and c' If `add_quotes` is True, each item in the list will be wrapped in single quotes. @@ -1719,7 +1690,7 @@ def comma_and(some_list: list | tuple, add_quotes=True) -> str: def comma_sep(some_list: list | tuple, pattern: str, add_quotes=True) -> str: - """Returns the given list or tuple as a comma separated string, with the last item joined by the given string format pattern. + """Return the given list or tuple as a comma separated string, with the last item joined by the given string format pattern. If `add_quotes` is True, each item in the list will be wrapped in single quotes. @@ -1740,7 +1711,7 @@ def comma_sep(some_list: list | tuple, pattern: str, add_quotes=True) -> str: def new_line_sep(some_list: list | tuple) -> str: - """Returns the given list or tuple as a new line separated string. + """Return the given list or tuple as a new line separated string. e.g. ['', 'Paid', 'Unpaid'] -> '\n Paid\n Unpaid' """ @@ -1828,7 +1799,7 @@ def get_host_name_from_request() -> str: def url_contains_port(url: str) -> bool: - """Returns True if the given url contains a port number. + """Return True if the given url contains a port number. e.g. 'http://localhost:8000' -> True, 'http://localhost' -> False """ @@ -1877,7 +1848,7 @@ def get_link_to_report( def get_absolute_url(doctype: str, name: str) -> str: - """Returns the absolute route for the form view of the given document in the desk. + """Return the absolute route for the form view of the given document in the desk. e.g. when doctype="Sales Invoice" and name="INV-00001", returns '/app/sales-invoice/INV-00001' """ @@ -1889,7 +1860,7 @@ def get_url_to_form(doctype: str, name: str) -> str: def get_url_to_list(doctype: str) -> str: - """Returns the absolute URL for the list view of the given document in the desk. + """Return the absolute URL for the list view of the given document in the desk. e.g. when doctype="Sales Invoice" and your site URL is "https://frappe.io", returns 'https://frappe.io/app/sales-invoice' @@ -1946,7 +1917,7 @@ operator_map = { def evaluate_filters(doc, filters: dict | list | tuple): - """Returns true if doc matches filters""" + """Return True if doc matches filters.""" if isinstance(filters, dict): for key, value in filters.items(): f = get_filter(None, {key: value}) @@ -1973,14 +1944,14 @@ def compare(val1: Any, condition: str, val2: Any, fieldtype: str | None = None): def get_filter(doctype: str, f: dict | list | tuple, filters_config=None) -> "frappe._dict": - """Returns a _dict like + """Return a `_dict` like: { - "doctype": - "fieldname": - "operator": - "value": - "fieldtype": + "doctype": ... + "fieldname": ... + "operator": ... + "value": ... + "fieldtype": ... } """ from frappe.database.utils import NestedSetHierarchy @@ -2181,7 +2152,7 @@ def get_string_between(start: str, string: str, end: str) -> str: def to_markdown(html: str) -> str: - """Converts the given HTML to markdown and returns it.""" + """Convert the given HTML to markdown and returns it.""" from html.parser import HTMLParser from frappe.core.utils import html2text @@ -2193,7 +2164,7 @@ def to_markdown(html: str) -> str: def md_to_html(markdown_text: str) -> Optional["UnicodeWithAttrs"]: - """Converts the given markdown text to HTML and returns it.""" + """Convert the given markdown text to HTML and returns it.""" from markdown2 import MarkdownError from markdown2 import markdown as _markdown @@ -2213,12 +2184,12 @@ def md_to_html(markdown_text: str) -> Optional["UnicodeWithAttrs"]: def markdown(markdown_text: str) -> Optional["UnicodeWithAttrs"]: - """Converts the given markdown text to HTML and returns it.""" + """Convert the given markdown text to HTML and returns it.""" return md_to_html(markdown_text) def is_subset(list_a: list, list_b: list) -> bool: - """Returns whether list_a is a subset of list_b""" + """Return whether list_a is a subset of list_b.""" return len(list(set(list_a) & set(list_b))) == len(list_a) @@ -2231,7 +2202,7 @@ def generate_hash(*args, **kwargs) -> str: def dict_with_keys(dict, keys): - """Returns a new dict with a subset of keys""" + """Return a new dict with a subset of keys.""" out = {} for key in dict: if key in keys: diff --git a/frappe/utils/file_lock.py b/frappe/utils/file_lock.py index cb86d2f3de..97d37eb940 100644 --- a/frappe/utils/file_lock.py +++ b/frappe/utils/file_lock.py @@ -37,7 +37,7 @@ def create_lock(name): def lock_exists(name): - """Returns True if lock of the given name exists""" + """Return True if lock of the given name exists.""" return os.path.exists(get_lock_path(name)) diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index c357c58371..05c05e1f1b 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -71,7 +71,7 @@ def upload(): def get_file_doc(dt=None, dn=None, folder=None, is_private=None, df=None): - """returns File object (Document) from given parameters or form_dict""" + """Return File object (Document) from given parameters or `form_dict`.""" r = frappe.form_dict if dt is None: @@ -328,7 +328,7 @@ def delete_file(path): def get_file(fname): - """Returns [`file_name`, `content`] for given file name `fname`""" + """Return [`file_name`, `content`] for given file name `fname`.""" file_path = get_file_path(fname) # read the file @@ -345,7 +345,7 @@ def get_file(fname): def get_file_path(file_name): - """Returns file path from given file name""" + """Return file path from given file name.""" if "../" in file_name: return diff --git a/frappe/utils/identicon.py b/frappe/utils/identicon.py index f7bc32bbd5..43517087d0 100644 --- a/frappe/utils/identicon.py +++ b/frappe/utils/identicon.py @@ -28,9 +28,7 @@ class Identicon: self.hash = self.digest(str_) def digest(self, str_): - """ - Returns a md5 numeric hash - """ + """Return a md5 numeric hash.""" return int(md5(str_.encode("utf-8"), usedforsecurity=False).hexdigest(), 16) def calculate(self): diff --git a/frappe/utils/image.py b/frappe/utils/image.py index eb57fbe0a7..055887dc0a 100644 --- a/frappe/utils/image.py +++ b/frappe/utils/image.py @@ -20,14 +20,13 @@ def resize_images(path, maxdim=700): print(f"resized {os.path.join(basepath, fname)}") -def strip_exif_data(content, content_type): - """Strips EXIF from image files which support it. +def strip_exif_data(content, content_type) -> bytes: + """Strip EXIF from image files which support it. Works by creating a new Image object which ignores exif by default and then extracts the binary data back into content. - Returns: - Bytes: Stripped image content + Return Stripped image content. """ original_image = Image.open(io.BytesIO(content)) diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index 694887395e..5ccfdf6ff6 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -154,7 +154,7 @@ def set_filters(jenv): def get_jinja_hooks(): - """Returns a tuple of (methods, filters) each containing a dict of method name and method definition pair.""" + """Return a tuple of (methods, filters) each containing a dict of method name and method definition pair.""" import frappe if not getattr(frappe.local, "site", None): diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 360c5eaedc..e7df115210 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -22,7 +22,7 @@ def get_logger( file_count=20, stream_only=stream_logging, ) -> "logging.Logger": - """Application Logger for your given module + """Return Application Logger for your given module. Args: module (str, optional): Name of your logger and consequently your log file. Defaults to None. @@ -33,8 +33,7 @@ def get_logger( file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20. stream_only (bool, optional): Whether to stream logs only to stderr (True) or use log files (False). Defaults to False. - Returns: - : Returns a Python logger object with Site and Bench level logging capabilities. + Return a Python logger object with Site and Bench level logging capabilities. """ if allow_site is True: diff --git a/frappe/utils/print_format.py b/frappe/utils/print_format.py index 7ab0c24a7c..e229f64b6d 100644 --- a/frappe/utils/print_format.py +++ b/frappe/utils/print_format.py @@ -22,11 +22,9 @@ from frappe.www.printview import validate_print_permission def download_multi_pdf( doctype, name, format=None, no_letterhead=False, letterhead=None, options=None ): - """ - Concatenate multiple docs as PDF . + """Return a PDF compiled by concatenating multiple documents. - Returns a PDF compiled by concatenating multiple documents. The documents - can be from a single DocType or multiple DocTypes + The documents can be from a single DocType or multiple DocTypes. Note: The design may seem a little weird, but it exists exists to ensure backward compatibility. The correct way to use this function is to @@ -43,9 +41,6 @@ def download_multi_pdf( format: Print Format to be used - Returns: - PDF: A PDF generated by the concatenation of the mentioned input docs - OLD FUNCTIONALITY - soon to be deprecated ========================================= Parameters: diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 45be0c63e8..bd82db4738 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -72,8 +72,8 @@ class RedisWrapper(redis.Redis): return None def get_value(self, key, generator=None, user=None, expires=False, shared=False): - """Returns cache value. If not found and generator function is - given, it will call the generator. + """Return cache value. If not found and generator function is + given, call the generator. :param key: Cache key. :param generator: Function to be called to generate a value if `None` is returned. @@ -266,23 +266,23 @@ class RedisWrapper(redis.Redis): super().sadd(self.make_key(name), *values) def srem(self, name, *values): - """Remove a specific member/list of members from the set""" + """Remove a specific member/list of members from the set.""" super().srem(self.make_key(name), *values) def sismember(self, name, value): - """Returns True or False based on if a given value is present in the set""" + """Return True or False based on if a given value is present in the set.""" return super().sismember(self.make_key(name), value) def spop(self, name): - """Removes and returns a random member from the set""" + """Remove and returns a random member from the set.""" return super().spop(self.make_key(name)) def srandmember(self, name, count=None): - """Returns a random member from the set""" + """Return a random member from the set.""" return super().srandmember(self.make_key(name)) def smembers(self, name): - """Return all members of the set""" + """Return all members of the set.""" return super().smembers(self.make_key(name)) def ft(self, index_name="idx"): diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index 6c9d36cb5c..8c048c1a16 100755 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -141,8 +141,8 @@ def disable_scheduler(): def schedule_jobs_based_on_activity(check_time=None): - """Returns True for active sites defined by Activity Log - Returns True for inactive sites once in 24 hours""" + """Return True for active sites defined by Activity Log. + Also return True for inactive sites once in 24 hours.""" if is_dormant(check_time=check_time): # ensure last job is one day old last_job_timestamp = _get_last_modified_timestamp("Scheduled Job Log") diff --git a/frappe/utils/user.py b/frappe/utils/user.py index d243c907fe..032e35ef55 100644 --- a/frappe/utils/user.py +++ b/frappe/utils/user.py @@ -285,7 +285,7 @@ def get_fullname_and_avatar(user: str) -> _dict: def get_system_managers(only_name: bool = False) -> list[str]: - """returns all system manager's user details""" + """Return all system manager's user details.""" HasRole = DocType("Has Role") User = DocType("User") diff --git a/frappe/utils/weasyprint.py b/frappe/utils/weasyprint.py index 8dff667bbd..8b633bf57c 100644 --- a/frappe/utils/weasyprint.py +++ b/frappe/utils/weasyprint.py @@ -103,12 +103,7 @@ class PrintFormatGenerator: return header_html, footer_html def render_pdf(self): - """ - Returns - ------- - pdf: a bytes sequence - The rendered PDF. - """ + """Return a bytes sequence of the rendered PDF.""" HTML, CSS = import_weasyprint() self._make_header_footer() diff --git a/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py b/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py index bd7d358bd8..7d64b77e60 100644 --- a/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py +++ b/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py @@ -70,7 +70,7 @@ class PersonalDataDownloadRequest(Document): def get_user_data(user): - """returns user data not linked to User doctype""" + """Return user data not linked to `User` doctype.""" hooks = frappe.get_hooks("user_data_fields") data = {} for hook in hooks: diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 2c325742d1..4d78b60483 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -107,7 +107,7 @@ class WebForm(WebsiteGenerator): frappe.throw(_("Following fields are missing:") + "
" + "
".join(missing)) def reset_field_parent(self): - """Convert link fields to select with names as options""" + """Convert link fields to select with names as options.""" for df in self.web_form_fields: df.parent = self.doc_type diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index 12f86f44c2..3e6d6cbc4c 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -235,7 +235,7 @@ def check_publish_status(): def get_web_blocks_html(blocks): - """Converts a list of blocks into Raw HTML and extracts out their scripts for deduplication""" + """Convert a list of blocks into Raw HTML and extract out their scripts for deduplication.""" out = frappe._dict(html="", scripts={}, styles={}) extracted_scripts = {} diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index 88560448c1..c50b20e71e 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -38,7 +38,7 @@ def authorize_access(reauthorize=False, code=None): def get_google_indexing_object(): - """Returns an object of Google Indexing object.""" + """Return an object of Google Indexing object.""" account = frappe.get_doc("Website Settings") oauth_obj = GoogleOAuth("indexing") diff --git a/frappe/website/path_resolver.py b/frappe/website/path_resolver.py index a91a51c0da..811977b451 100644 --- a/frappe/website/path_resolver.py +++ b/frappe/website/path_resolver.py @@ -24,7 +24,7 @@ class PathResolver: self.http_status_code = http_status_code def resolve(self): - """Returns endpoint and a renderer instance that can render the endpoint""" + """Return endpoint and a renderer instance that can render the endpoint.""" request = frappe._dict() if hasattr(frappe.local, "request"): request = frappe.local.request or request diff --git a/frappe/website/router.py b/frappe/website/router.py index 2947b89f91..3bffa78fee 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -254,8 +254,8 @@ def setup_source(page_info): def get_base_template(path=None): - """ - Returns the `base_template` for given `path`. + """Return the `base_template` for given `path`. + The default `base_template` for any web route is `templates/web.html` defined in `hooks.py`. This can be overridden for certain routes in `custom_app/hooks.py` based on regex pattern. """ diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 113bef049c..568d11777d 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -251,7 +251,7 @@ def get_next_link(route, url_prefix=None, app=None): def get_full_index(route=None, app=None): - """Returns full index of the website for www upto the n-th level""" + """Return full index of the website for www upto the n-th level.""" from frappe.website.router import get_pages if not frappe.local.flags.children_map: @@ -305,7 +305,7 @@ def get_full_index(route=None, app=None): def extract_title(source, path): - """Returns title from `<!-- title -->` or <h1> or path""" + """Return title from `<!-- title -->` or <h1> or path.""" title = extract_comment_tag(source, "title") if not title and "

" in source: diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py index 8f814f60cb..598978dd46 100644 --- a/frappe/website/website_generator.py +++ b/frappe/website/website_generator.py @@ -45,8 +45,7 @@ class WebsiteGenerator(Document): self.route = self.route.strip("/.")[:139] def make_route(self): - """Returns the default route. If `route` is specified in DocType it will be - route/title""" + """Return the default route. If `route` is specified in DocType it will be route/title.""" from_title = self.scrubbed_title() if self.meta.route: return self.meta.route + "/" + from_title diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index 382a67cd8a..9cf61126b4 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -496,9 +496,7 @@ def get_common_email_args(doc): def get_email_template(doc): - """Returns next_action_email_template - for workflow state (if available) based on doc current workflow state - """ + """Return next_action_email_template for workflow state (if available) based on doc current workflow state.""" workflow_name = get_workflow_name(doc.get("doctype")) doc_state = get_doc_workflow_state(doc) template_name = frappe.db.get_value( diff --git a/frappe/www/list.py b/frappe/www/list.py index bd0db9fd05..aaf20f8a05 100644 --- a/frappe/www/list.py +++ b/frappe/www/list.py @@ -13,8 +13,9 @@ no_cache = 1 def get_context(context, **dict_params): - """Returns context for a list standard list page. - Will also update `get_list_context` from the doctype module file""" + """Return context for a list standard list page. + + Also update `get_list_context` from the doctype module file.""" frappe.local.form_dict.update(dict_params) doctype = frappe.local.form_dict.doctype context.parents = [{"route": "me", "title": _("My Account")}] @@ -27,7 +28,7 @@ def get_context(context, **dict_params): @frappe.whitelist(allow_guest=True) def get(doctype, txt=None, limit_start=0, limit=20, pathname=None, **kwargs): - """Returns processed HTML page for a standard listing.""" + """Return processed HTML page for a standard listing.""" limit_start = cint(limit_start) raw_result = get_list_data(doctype, txt, limit_start, limit=limit + 1, **kwargs) show_more = len(raw_result) > limit @@ -77,7 +78,7 @@ def get(doctype, txt=None, limit_start=0, limit=20, pathname=None, **kwargs): def get_list_data( doctype, txt=None, limit_start=0, fields=None, cmd=None, limit=20, web_form_name=None, **kwargs ): - """Returns processed HTML page for a standard listing.""" + """Return processed HTML page for a standard listing.""" limit_start = cint(limit_start) if frappe.is_table(doctype): diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 7d7798864d..da08d2cd45 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -79,7 +79,7 @@ def get_context(context): def get_print_format_doc(print_format_name, meta): - """Returns print format document""" + """Return print format document.""" if not print_format_name: print_format_name = frappe.form_dict.format or meta.default_print_format or "Standard" @@ -265,7 +265,7 @@ def set_title_values_for_table_and_multiselect_fields(meta, doc): def convert_markdown(doc: "Document"): - """Convert text field values to markdown if necessary""" + """Convert text field values to markdown if necessary.""" for field in doc.meta.fields: if field.fieldtype == "Text Editor": value = doc.get(field.fieldname) @@ -284,7 +284,7 @@ def get_html_and_style( style: str | None = None, settings: str | None = None, ): - """Returns `html` and `style` of print format, used in PDF etc""" + """Return `html` and `style` of print format, used in PDF etc.""" if isinstance(name, str): document = frappe.get_doc(doc, name) @@ -315,7 +315,7 @@ def get_html_and_style( @frappe.whitelist() def get_rendered_raw_commands(doc: str, name: str | None = None, print_format: str | None = None): - """Returns Rendered Raw Commands of print format, used to send directly to printer""" + """Return Rendered Raw Commands of print format, used to send directly to printer.""" if isinstance(name, str): document = frappe.get_doc(doc, name) @@ -500,7 +500,7 @@ def make_layout(doc, meta, format_data=None): def is_visible(df, doc): - """Returns True if docfield is visible in print layout and does not have print_hide set.""" + """Return True if docfield is visible in print layout and does not have print_hide set.""" if df.fieldtype in ("Section Break", "Column Break", "Button"): return False @@ -580,7 +580,7 @@ def get_font(print_settings, print_format=None, for_legacy=False): def get_visible_columns(data, table_meta, df): - """Returns list of visible columns based on print_hide and if all columns have value.""" + """Return list of visible columns based on print_hide and if all columns have value.""" columns = [] doc = data[0] or frappe.new_doc(df.options) diff --git a/frappe/www/sitemap.py b/frappe/www/sitemap.py index dbf1f3a5f3..ebe6846a39 100644 --- a/frappe/www/sitemap.py +++ b/frappe/www/sitemap.py @@ -32,7 +32,7 @@ def get_context(context): def get_public_pages_from_doctypes(): - """Returns pages from doctypes that are publicly accessible""" + """Return pages from doctypes that are publicly accessible.""" def get_sitemap_routes(): routes = {} From caae0bacc9f58996a58bb63d158b9703d0de1492 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 19:09:45 +0530 Subject: [PATCH 171/237] docs: add missing full stop --- frappe/build.py | 2 +- frappe/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/build.py b/frappe/build.py index 8d66846b57..af9bfce72d 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -418,7 +418,7 @@ def scrub_html_template(content): def html_to_js_template(path, content): - """Return HTML template content as Javascript code, by adding it to `frappe.templates`""" + """Return HTML template content as Javascript code, by adding it to `frappe.templates`.""" return """frappe.templates["{key}"] = '{content}';\n""".format( key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content) ) diff --git a/frappe/client.py b/frappe/client.py index 2d7e765564..028df862c4 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -354,7 +354,7 @@ def get_js(items): @frappe.whitelist(allow_guest=True) def get_time_zone(): - """Return the default time zone""" + """Return the default time zone.""" return {"time_zone": frappe.defaults.get_defaults().get("time_zone")} From 7e821a131a3b2c9d621148b8cb617ec45dd28554 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 18 Dec 2023 19:41:41 +0530 Subject: [PATCH 172/237] fix: allow enabling telemetry for either posthog/sentry (#23845) posthog is not required to be present everwhere and is more of a solution for onboarding problems. Sentry on other hand should be available everywhere. --- frappe/public/js/telemetry/index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/telemetry/index.js b/frappe/public/js/telemetry/index.js index 4c71b18f28..d15101b3d1 100644 --- a/frappe/public/js/telemetry/index.js +++ b/frappe/public/js/telemetry/index.js @@ -43,7 +43,12 @@ class TelemetryManager { } can_enable() { - return Boolean(this.telemetry_host && this.project_id && !cint(navigator.doNotTrack)); + if (cint(navigator.doNotTrack)) { + return false; + } + let posthog_available = Boolean(this.telemetry_host && this.project_id); + let sentry_available = Boolean(frappe.boot.sentry_dsn); + return posthog_available || sentry_available; } send_heartbeat() { From 541e99e1c8e204a4b8c68a5c79f2fab1ff6f8a5f Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 21:49:13 +0530 Subject: [PATCH 173/237] refactor: only import types when TYPE_CHECKING --- frappe/utils/password_strength.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/frappe/utils/password_strength.py b/frappe/utils/password_strength.py index 4f24bc61df..253421a6d4 100644 --- a/frappe/utils/password_strength.py +++ b/frappe/utils/password_strength.py @@ -1,18 +1,23 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from collections.abc import Iterable +from typing import TYPE_CHECKING -from zxcvbn import _Result, zxcvbn -from zxcvbn.feedback import _Feedback as PasswordStrengthFeedback -from zxcvbn.matching import _Match +from zxcvbn import zxcvbn from zxcvbn.scoring import ALL_UPPER, START_UPPER import frappe from frappe import _ +if TYPE_CHECKING: + from collections.abc import Iterable -def test_password_strength(password: str, user_inputs: Iterable[object] = None) -> _Result: + from zxcvbn import _Result + from zxcvbn.feedback import _Feedback as PasswordStrengthFeedback + from zxcvbn.matching import _Match + + +def test_password_strength(password: str, user_inputs: "Iterable[object]" = None) -> "_Result": """Wrapper around zxcvbn.password_strength""" if len(password) > 128: # zxcvbn takes forever when checking long, random passwords. @@ -33,7 +38,7 @@ def test_password_strength(password: str, user_inputs: Iterable[object] = None) # Default feedback value -default_feedback: PasswordStrengthFeedback = { +default_feedback: "PasswordStrengthFeedback" = { "warning": "", "suggestions": [ _("Use a few words, avoid common phrases."), @@ -42,7 +47,7 @@ default_feedback: PasswordStrengthFeedback = { } -def get_feedback(score: int, sequence: list) -> PasswordStrengthFeedback: +def get_feedback(score: int, sequence: list) -> "PasswordStrengthFeedback": """Return the feedback dictionary consisting of ("warning","suggestions") for the given sequences.""" global default_feedback minimum_password_score = int( @@ -72,7 +77,7 @@ def get_feedback(score: int, sequence: list) -> PasswordStrengthFeedback: return feedback -def get_match_feedback(match: _Match, is_sole_match: bool) -> PasswordStrengthFeedback: +def get_match_feedback(match: "_Match", is_sole_match: bool) -> "PasswordStrengthFeedback": """Return feedback as a dictionary for a certain match.""" def fun_bruteforce(): @@ -143,7 +148,9 @@ def get_match_feedback(match: _Match, is_sole_match: bool) -> PasswordStrengthFe return pattern_fn() -def get_dictionary_match_feedback(match: _Match, is_sole_match: bool) -> PasswordStrengthFeedback: +def get_dictionary_match_feedback( + match: "_Match", is_sole_match: bool +) -> "PasswordStrengthFeedback": """Return feedback for a match that is found in a dictionary.""" warning = "" suggestions = [] From 8f7638860f56e1438e0973956b3633726d03bf61 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 21:55:55 +0530 Subject: [PATCH 174/237] docs: remainder, image_to_base64, get_url, get_host_name*, get_link_to_report*, get_url_to_form --- frappe/utils/data.py | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index dd92d72318..c5eda50136 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -47,6 +47,8 @@ TimespanOptions = Literal[ if typing.TYPE_CHECKING: + from PIL.ImageFile import ImageFile as PILImageFile + T = TypeVar("T") @@ -1280,6 +1282,7 @@ def _bankers_rounding(num, precision): def remainder(numerator: NumericType, denominator: NumericType, precision: int = 2) -> NumericType: + """Return the remainder of the division of `numerator` by `denominator`.""" precision = cint(precision) multiplier = 10**precision @@ -1601,7 +1604,8 @@ def get_thumbnail_base64_for_image(src): return cache().hget("thumbnail_base64", src, generator=_get_base64) -def image_to_base64(image, extn: str) -> bytes: +def image_to_base64(image: "PILImageFile", extn: str) -> bytes: + """Return the base64 encoded string for the given PIL `ImageFile`.""" from io import BytesIO buffered = BytesIO() @@ -1735,7 +1739,7 @@ def filter_strip_join(some_list: list[str], sep: str) -> list[str]: def get_url(uri: str | None = None, full_address: bool = False) -> str: - """get app url from request""" + """Get app url from request.""" host_name = frappe.local.conf.host_name or frappe.local.conf.hostname if uri and (uri.startswith("http://") or uri.startswith("https://")): @@ -1791,6 +1795,7 @@ def get_url(uri: str | None = None, full_address: bool = False) -> str: def get_host_name_from_request() -> str: + """Return the hostname (`request.host`) from the request headers.""" if hasattr(frappe.local, "request") and frappe.local.request and frappe.local.request.host: protocol = ( "https://" if "https" == frappe.get_request_header("X-Forwarded-Proto", "") else "http://" @@ -1801,13 +1806,17 @@ def get_host_name_from_request() -> str: def url_contains_port(url: str) -> bool: """Return True if the given url contains a port number. - e.g. 'http://localhost:8000' -> True, 'http://localhost' -> False + e.g. 'http://localhost:8000' -> True, 'http://localhost' -> False. """ parts = url.split(":") return len(parts) > 2 def get_host_name() -> str: + """Return the hostname of the current site. + + e.g. If site is 'https://cloud.frappe.io', returns 'cloud.frappe.io'. + """ return get_url().rsplit("//", 1)[-1] @@ -1825,6 +1834,12 @@ def get_link_to_report( doctype: str | None = None, filters: dict | None = None, ) -> str: + """ + Return the HTML link to the given report. + + e.g. get_link_to_report("Revenue Report", "Link Label") returns: + "Link Label". + """ if not label: label = name @@ -1856,6 +1871,11 @@ def get_absolute_url(doctype: str, name: str) -> str: def get_url_to_form(doctype: str, name: str) -> str: + """Return the absolute URL for the form view of the given document in the desk. + + e.g. when doctype="Sales Invoice" and your site URL is "https://frappe.io", + returns 'https://frappe.io/app/sales-invoice/INV-00001' + """ return get_url(uri=f"/app/{quoted(slug(doctype))}/{quoted(name)}") @@ -1869,6 +1889,15 @@ def get_url_to_list(doctype: str) -> str: def get_url_to_report(name, report_type: str | None = None, doctype: str | None = None) -> str: + """Return the absolute URL for the report in the desk. + + e.g. when name="Sales Register" and your site URL is "https://frappe.io", + returns 'https://frappe.io/app/query-report/Sales%20Register' + + You can optionally pass `report_type` and `doctype` to get the URL for a Report Builder report. + + get_url_to_report("Revenue", "Report Builder", "Sales Invoice") -> 'https://frappe.io/app/sales-invoice/view/report/Revenue' + """ if report_type == "Report Builder": return get_url(uri=f"/app/{quoted(slug(doctype))}/view/report/{quoted(name)}") else: @@ -1876,6 +1905,7 @@ def get_url_to_report(name, report_type: str | None = None, doctype: str | None def get_url_to_report_with_filters(name, filters, report_type=None, doctype=None): + """Return the absolute URL for the report in the desk with filters.""" if report_type == "Report Builder": return get_url(uri=f"/app/{quoted(slug(doctype))}/view/report?{filters}") @@ -2305,6 +2335,14 @@ class _UserInfo(typing.TypedDict): def get_user_info_for_avatar(user_id: str) -> _UserInfo: + """Return user info for the given `user_id` suitable for use in an avatar. + + e.g. { + "email": "faris@frappe.io", + "image": "/assets/frappe/images/ui/avatar.png", + "name": "Faris Ansari" + } + """ try: user = frappe.get_cached_doc("User", user_id) return {"email": user.email, "image": user.user_image, "name": user.full_name} From cc9a866142d620b0cd80fd3ca6f5e70d1fb64fcf Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 22:00:44 +0530 Subject: [PATCH 175/237] docs: get_thumbnail_base64_for_image --- frappe/utils/data.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index c5eda50136..1feaeab0aa 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1563,7 +1563,17 @@ def is_image(filepath: str) -> bool: return (guess_type(filepath)[0] or "").startswith("image/") -def get_thumbnail_base64_for_image(src): +def get_thumbnail_base64_for_image(src: str) -> dict[str, str] | None: + """Return the base64 encoded string for the thumbnail of the given image source path. + + Example return value: + + { + "base64": "data:image/ext;base64,...", + "width": 50, + "height": 50 + } + """ from os.path import exists as file_exists from PIL import Image From a22451c6e2d511d94437c9f5c92da54a285d6725 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 22:03:37 +0530 Subject: [PATCH 176/237] docs: pdf_to_base64 --- frappe/utils/data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 1feaeab0aa..7728895d20 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1626,6 +1626,9 @@ def image_to_base64(image: "PILImageFile", extn: str) -> bytes: def pdf_to_base64(filename: str) -> bytes | None: + """Return the base64 encoded string for the given PDF file. + + Return None if the file is not found or is not a PDF file.""" from frappe.utils.file_manager import get_file_path if "../" in filename or filename.rsplit(".")[-1] not in ["pdf", "PDF"]: From 3d35ee3e9e86a9ae5636c33c82b9c00b70c15ce6 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 22:08:35 +0530 Subject: [PATCH 177/237] docs: get_link_to_form --- frappe/utils/data.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 7728895d20..743ca92b3a 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1649,7 +1649,7 @@ _striptags_re = re.compile(r"(|<[^>]*>)") def strip_html(text: str) -> str: - """removes anything enclosed in and including <>""" + """Remove anything enclosed in and including <>.""" return _striptags_re.sub("", text) @@ -1669,8 +1669,7 @@ def escape_html(text: str) -> str: def pretty_date(iso_datetime: datetime.datetime | str) -> str: - """ - Return a localized string representation of the delta to the current system time. + """Return a localized string representation of the delta to the current system time. For example, "1 hour ago", "2 days ago", "in 5 seconds", etc. """ @@ -1834,6 +1833,11 @@ def get_host_name() -> str: def get_link_to_form(doctype: str, name: str, label: str | None = None) -> str: + """Return the HTML link to the given document's form view. + + e.g. get_link_to_form("Sales Invoice", "INV-0001", "Link Label") returns: + 'Link Label'. + """ if not label: label = name @@ -1847,11 +1851,10 @@ def get_link_to_report( doctype: str | None = None, filters: dict | None = None, ) -> str: - """ - Return the HTML link to the given report. + """Return the HTML link to the given report. e.g. get_link_to_report("Revenue Report", "Link Label") returns: - "Link Label". + 'Link Label'. """ if not label: label = name From 370d575f29b25d73880e3c4509f54c98dfdb8cb4 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 18 Dec 2023 22:27:58 +0530 Subject: [PATCH 178/237] docs: some more utils --- frappe/__init__.py | 1 + frappe/utils/data.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/frappe/__init__.py b/frappe/__init__.py index 4d8ededeb9..1c228fc2d3 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1152,6 +1152,7 @@ def set_value(doctype, docname, fieldname, value=None): def get_cached_doc(*args, **kwargs) -> "Document": + """Like `frappe.get_doc`, but return from cache if available.""" if (key := can_cache_doc(args)) and (doc := cache.get_value(key)): return doc diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 743ca92b3a..ddd1025abe 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1654,6 +1654,10 @@ def strip_html(text: str) -> str: def escape_html(text: str) -> str: + """Return the given text with HTML special characters escaped. + + e.g. '

Hello

' -> '<h1>Hello</h1>' + """ if not isinstance(text, str): return text @@ -2133,10 +2137,20 @@ def sanitize_column(column_name: str) -> None: def scrub_urls(html: str) -> str: + """Expand relative urls in the given `html`. + + e.g. If HTML is 'View Image' and site URL is 'https://frappe.io', + returns 'View Image'. + """ return expand_relative_urls(html) def expand_relative_urls(html: str) -> str: + """Expand relative urls in the given `html`. + + e.g. If HTML is 'View Image' and site URL is 'https://frappe.io', + returns 'View Image'. + """ # expand relative urls url = get_url() if url.endswith("/"): @@ -2162,6 +2176,10 @@ def expand_relative_urls(html: str) -> str: def quoted(url: str) -> str: + """Return the given `url` quoted. + + e.g. 'https://frappe.io/files/my Image file.jpeg' -> 'https://frappe.io/files/my%20Image%20file.jpeg' + """ return cstr(quote(encode(cstr(url)), safe=b"~@#$&()*!+=:;,.?/'")) @@ -2184,6 +2202,10 @@ def unique(seq: typing.Sequence["T"]) -> list["T"]: def strip(val: str, chars: str | None = None) -> str: + """Strip the given characters from the given string. + + e.g. strip(',hello,bye,', ',') -> 'hello,bye' + """ # \ufeff is no-width-break, \u200b is no-width-space return (val or "").replace("\ufeff", "").replace("\u200b", "").strip(chars) From 99f0745302771177e4cf8ae83d89a61379c1444d Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Mon, 18 Dec 2023 18:43:13 +0100 Subject: [PATCH 179/237] style: new app boilerplate (#23847) * style: add spaces to hooks boilerplate Black wants comments to start with a space. Adding them here means less formatting later. * style: init_template Put version on the first line and use double quotes. --- frappe/utils/boilerplate.py | 100 ++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 80edc67a0f..5a721279bd 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -303,9 +303,7 @@ class PatchCreator: init_py.touch() -init_template = """ -__version__ = '0.0.1' - +init_template = """__version__ = "0.0.1" """ pyproject_template = """[project] @@ -378,7 +376,7 @@ app_license = "{app_license}" # website user home page (by Role) # role_home_page = {{ -# "Role": "home_page" +# "Role": "home_page" # }} # Generators @@ -392,8 +390,8 @@ app_license = "{app_license}" # add methods and filters to jinja environment # jinja = {{ -# "methods": "{app_name}.utils.jinja_methods", -# "filters": "{app_name}.utils.jinja_filters" +# "methods": "{app_name}.utils.jinja_methods", +# "filters": "{app_name}.utils.jinja_filters" # }} # Installation @@ -435,11 +433,11 @@ app_license = "{app_license}" # Permissions evaluated in scripted ways # permission_query_conditions = {{ -# "Event": "frappe.desk.doctype.event.event.get_permission_query_conditions", +# "Event": "frappe.desk.doctype.event.event.get_permission_query_conditions", # }} # # has_permission = {{ -# "Event": "frappe.desk.doctype.event.event.has_permission", +# "Event": "frappe.desk.doctype.event.event.has_permission", # }} # DocType Class @@ -447,7 +445,7 @@ app_license = "{app_license}" # Override standard doctype classes # override_doctype_class = {{ -# "ToDo": "custom_app.overrides.CustomToDo" +# "ToDo": "custom_app.overrides.CustomToDo" # }} # Document Events @@ -455,32 +453,32 @@ app_license = "{app_license}" # Hook on document methods and events # doc_events = {{ -# "*": {{ -# "on_update": "method", -# "on_cancel": "method", -# "on_trash": "method" -# }} +# "*": {{ +# "on_update": "method", +# "on_cancel": "method", +# "on_trash": "method" +# }} # }} # Scheduled Tasks # --------------- # scheduler_events = {{ -# "all": [ -# "{app_name}.tasks.all" -# ], -# "daily": [ -# "{app_name}.tasks.daily" -# ], -# "hourly": [ -# "{app_name}.tasks.hourly" -# ], -# "weekly": [ -# "{app_name}.tasks.weekly" -# ], -# "monthly": [ -# "{app_name}.tasks.monthly" -# ], +# "all": [ +# "{app_name}.tasks.all" +# ], +# "daily": [ +# "{app_name}.tasks.daily" +# ], +# "hourly": [ +# "{app_name}.tasks.hourly" +# ], +# "weekly": [ +# "{app_name}.tasks.weekly" +# ], +# "monthly": [ +# "{app_name}.tasks.monthly" +# ], # }} # Testing @@ -492,14 +490,14 @@ app_license = "{app_license}" # ------------------------------ # # override_whitelisted_methods = {{ -# "frappe.desk.doctype.event.event.get_events": "{app_name}.event.get_events" +# "frappe.desk.doctype.event.event.get_events": "{app_name}.event.get_events" # }} # # each overriding function accepts a `data` argument; # generated from the base implementation of the doctype dashboard, # along with any modifications made in other Frappe apps # override_doctype_dashboards = {{ -# "Task": "{app_name}.task.get_dashboard_data" +# "Task": "{app_name}.task.get_dashboard_data" # }} # exempt linked doctypes from being automatically cancelled @@ -525,38 +523,38 @@ app_license = "{app_license}" # -------------------- # user_data_fields = [ -# {{ -# "doctype": "{{doctype_1}}", -# "filter_by": "{{filter_by}}", -# "redact_fields": ["{{field_1}}", "{{field_2}}"], -# "partial": 1, -# }}, -# {{ -# "doctype": "{{doctype_2}}", -# "filter_by": "{{filter_by}}", -# "partial": 1, -# }}, -# {{ -# "doctype": "{{doctype_3}}", -# "strict": False, -# }}, -# {{ -# "doctype": "{{doctype_4}}" -# }} +# {{ +# "doctype": "{{doctype_1}}", +# "filter_by": "{{filter_by}}", +# "redact_fields": ["{{field_1}}", "{{field_2}}"], +# "partial": 1, +# }}, +# {{ +# "doctype": "{{doctype_2}}", +# "filter_by": "{{filter_by}}", +# "partial": 1, +# }}, +# {{ +# "doctype": "{{doctype_3}}", +# "strict": False, +# }}, +# {{ +# "doctype": "{{doctype_4}}" +# }} # ] # Authentication and authorization # -------------------------------- # auth_hooks = [ -# "{app_name}.auth.validate" +# "{app_name}.auth.validate" # ] # Automatically update python controller files with type annotations for this app. # export_python_type_annotations = True # default_log_clearing_doctypes = {{ -# "Logging DocType Name": 30 # days to retain logs +# "Logging DocType Name": 30 # days to retain logs # }} """ From 3820926ffaddd3fa513b236c805eb9631684361c Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 19 Dec 2023 12:08:28 +0530 Subject: [PATCH 180/237] fix: dont pass removed parameter `name` --- frappe/www/printview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 7d7798864d..d3ac4cf98b 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -333,7 +333,7 @@ def get_rendered_raw_commands(doc: str, name: str | None = None, print_format: s return { "raw_commands": get_rendered_template( - doc=document, name=name, print_format=print_format, meta=document.meta + doc=document, print_format=print_format, meta=document.meta ) } From 4d20ef92007f98e230af2f32a032807f6b4d4fa8 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Tue, 19 Dec 2023 12:08:50 +0530 Subject: [PATCH 181/237] docs: get_gravatar and get_gravatar_url --- frappe/utils/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 3e661978a6..b0db0c7b91 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -270,6 +270,11 @@ def has_gravatar(email: str) -> str: def get_gravatar_url(email: str, default: Literal["mm", "404"] = "mm") -> str: + """Return gravatar URL for the given email. + + If `default` is set to "404", gravatar URL will return 404 if no avatar is found. + If `default` is set to "mm", a placeholder image will be returned. + """ hexdigest = hashlib.md5( frappe.as_unicode(email).encode("utf-8"), usedforsecurity=False ).hexdigest() @@ -277,6 +282,9 @@ def get_gravatar_url(email: str, default: Literal["mm", "404"] = "mm") -> str: def get_gravatar(email: str) -> str: + """Return gravatar URL if user has set an avatar at gravatar.com. + + Else return identicon image (base64).""" from frappe.utils.identicon import Identicon return has_gravatar(email) or Identicon(email).base64() From 0df45daacd054cd93f6553649a8f363e590f4602 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Tue, 19 Dec 2023 12:17:00 +0530 Subject: [PATCH 182/237] docs: get_number_format_info --- frappe/utils/data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index ddd1025abe..016c62ba79 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1452,6 +1452,12 @@ number_format_info = { def get_number_format_info(format: str) -> tuple[str, str, int]: + """Return the decimal separator, thousands separator and precision for the given number `format` string. + + e.g. get_number_format_info('1,00,000.50') -> ('.', ',', 2) + + Will return ('.', ',', 2) for format strings which can't be guessed. + """ return number_format_info.get(format) or (".", ",", 2) From 3520bcb9b31378e6189723a239f761cdffb6b7d5 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Tue, 19 Dec 2023 12:55:02 +0530 Subject: [PATCH 183/237] fix: add (more) type hints printview.py --- frappe/www/printview.py | 74 +++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index da08d2cd45..79cb500609 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -5,7 +5,7 @@ import copy import json import os import re -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, TypedDict import frappe from frappe import _, get_module_path @@ -15,15 +15,30 @@ from frappe.utils import cint, escape_html, strip_html from frappe.utils.jinja_globals import is_rtl if TYPE_CHECKING: + from frappe.core.doctype.docfield.docfield import DocField from frappe.model.document import Document + from frappe.model.meta import Meta from frappe.printing.doctype.print_format.print_format import PrintFormat + from frappe.printing.doctype.print_settings.print_settings import PrintSettings no_cache = 1 standard_format = "templates/print_formats/standard.html" -def get_context(context): +class PrintContext(TypedDict): + body: str + print_style: str + comment: str + title: str + lang: str + layout_direction: str + doctype: str + name: str + key: str + + +def get_context(context) -> PrintContext: """Build context for print""" if not ((frappe.form_dict.doctype and frappe.form_dict.name) or frappe.form_dict.doc): return { @@ -78,7 +93,7 @@ def get_context(context): } -def get_print_format_doc(print_format_name, meta): +def get_print_format_doc(print_format_name: str, meta: "Meta") -> "PrintFormat" | None: """Return print format document.""" if not print_format_name: print_format_name = frappe.form_dict.format or meta.default_print_format or "Standard" @@ -95,13 +110,13 @@ def get_print_format_doc(print_format_name, meta): def get_rendered_template( doc: "Document", - print_format: str | None = None, - meta=None, + print_format: "PrintFormat" | None = None, + meta: "Meta" = None, no_letterhead: bool | None = None, letterhead: str | None = None, trigger_print: bool = False, - settings=None, -): + settings: dict = None, +) -> str: print_settings = frappe.get_single("Print Settings").as_dict() print_settings.update(settings or {}) @@ -219,7 +234,7 @@ def get_rendered_template( return html -def set_link_titles(doc): +def set_link_titles(doc: "Document") -> None: # Adds name with title of link field doctype to __link_titles if not doc.get("__link_titles"): setattr(doc, "__link_titles", {}) @@ -229,7 +244,9 @@ def set_link_titles(doc): set_title_values_for_table_and_multiselect_fields(meta, doc) -def set_title_values_for_link_and_dynamic_link_fields(meta, doc, parent_doc=None): +def set_title_values_for_link_and_dynamic_link_fields( + meta: "Meta", doc: "Document", parent_doc: Optional["Document"] = None +) -> None: if parent_doc and not parent_doc.get("__link_titles"): setattr(parent_doc, "__link_titles", {}) elif doc and not doc.get("__link_titles"): @@ -254,7 +271,7 @@ def set_title_values_for_link_and_dynamic_link_fields(meta, doc, parent_doc=None doc.__link_titles[f"{doctype}::{doc.get(field.fieldname)}"] = link_title -def set_title_values_for_table_and_multiselect_fields(meta, doc): +def set_title_values_for_table_and_multiselect_fields(meta: "Meta", doc: "Document") -> None: for field in meta.get_table_fields(): if not doc.get(field.fieldname): continue @@ -264,7 +281,7 @@ def set_title_values_for_table_and_multiselect_fields(meta, doc): set_title_values_for_link_and_dynamic_link_fields(_meta, value, doc) -def convert_markdown(doc: "Document"): +def convert_markdown(doc: "Document") -> None: """Convert text field values to markdown if necessary.""" for field in doc.meta.fields: if field.fieldtype == "Text Editor": @@ -283,7 +300,7 @@ def get_html_and_style( trigger_print: bool = False, style: str | None = None, settings: str | None = None, -): +) -> dict[str, str]: """Return `html` and `style` of print format, used in PDF etc.""" if isinstance(name, str): @@ -314,7 +331,9 @@ def get_html_and_style( @frappe.whitelist() -def get_rendered_raw_commands(doc: str, name: str | None = None, print_format: str | None = None): +def get_rendered_raw_commands( + doc: str, name: str | None = None, print_format: str | None = None +) -> dict: """Return Rendered Raw Commands of print format, used to send directly to printer.""" if isinstance(name, str): @@ -338,7 +357,7 @@ def get_rendered_raw_commands(doc: str, name: str | None = None, print_format: s } -def validate_print_permission(doc): +def validate_print_permission(doc: "Document") -> None: for ptype in ("read", "print"): if frappe.has_permission(doc.doctype, ptype, doc) or frappe.has_website_permission(doc): return @@ -350,7 +369,7 @@ def validate_print_permission(doc): raise frappe.PermissionError(_("You do not have permission to view this document")) -def validate_key(key, doc): +def validate_key(key: str, doc: "Document") -> None: document_key_expiry = frappe.get_cached_value( "Document Share Key", {"reference_doctype": doc.doctype, "reference_docname": doc.name, "key": key}, @@ -369,7 +388,9 @@ def validate_key(key, doc): raise frappe.exceptions.InvalidKeyError -def get_letter_head(doc: "Document", no_letterhead: bool, letterhead: str | None = None): +def get_letter_head( + doc: "Document", no_letterhead: bool, letterhead: str | None = None +) -> dict | "frappe._dict": if no_letterhead: return {} if letterhead: @@ -382,7 +403,7 @@ def get_letter_head(doc: "Document", no_letterhead: bool, letterhead: str | None ) -def get_print_format(doctype, print_format): +def get_print_format(doctype: str, print_format: "PrintFormat") -> str: if print_format.disabled: frappe.throw( _("Print Format {0} is disabled").format(print_format.name), frappe.DoesNotExistError @@ -407,7 +428,7 @@ def get_print_format(doctype, print_format): frappe.throw(_("No template found at path: {0}").format(path), frappe.TemplateNotFoundError) -def make_layout(doc, meta, format_data=None): +def make_layout(doc: "Document", meta: "Meta", format_data=None) -> list: """Builds a hierarchical layout object from the fields list to be rendered by `standard.html` @@ -499,7 +520,7 @@ def make_layout(doc, meta, format_data=None): return layout -def is_visible(df, doc): +def is_visible(df: "DocField", doc: "Document") -> bool: """Return True if docfield is visible in print layout and does not have print_hide set.""" if df.fieldtype in ("Section Break", "Column Break", "Button"): return False @@ -510,7 +531,8 @@ def is_visible(df, doc): return not doc.is_print_hide(df.fieldname, df) -def has_value(df, doc): +def has_value(df: "DocField", doc: "Document") -> bool: + """Return True if given docfield (`df`) has some value in the given document (`doc`).""" value = doc.get(df.fieldname) if value in (None, ""): return False @@ -529,7 +551,7 @@ def has_value(df, doc): def get_print_style( style: str | None = None, print_format: Optional["PrintFormat"] = None, for_legacy: bool = False -): +) -> str: print_settings = frappe.get_doc("Print Settings") if not style: @@ -559,7 +581,9 @@ def get_print_style( return css -def get_font(print_settings, print_format=None, for_legacy=False): +def get_font( + print_settings: "PrintSettings", print_format: Optional["PrintFormat"] = None, for_legacy=False +) -> str: default = 'Inter, "Helvetica Neue", Helvetica, Arial, "Open Sans", sans-serif' if for_legacy: return default @@ -579,14 +603,14 @@ def get_font(print_settings, print_format=None, for_legacy=False): return font -def get_visible_columns(data, table_meta, df): +def get_visible_columns(data: list, table_meta: "Meta", df: "DocField") -> list["DocField"]: """Return list of visible columns based on print_hide and if all columns have value.""" columns = [] doc = data[0] or frappe.new_doc(df.options) hide_in_print_layout = df.get("hide_in_print_layout") or [] - def add_column(col_df): + def add_column(col_df: "DocField"): if col_df.fieldname in hide_in_print_layout: return False return is_visible(col_df, doc) and column_has_value(data, col_df.get("fieldname"), col_df) @@ -610,7 +634,7 @@ def get_visible_columns(data, table_meta, df): return columns -def column_has_value(data, fieldname, col_df): +def column_has_value(data: list, fieldname: str, col_df: "DocField") -> bool: """Check if at least one cell in column has non-zero and non-blank value""" has_value = False From c70229cd69ece81a8eddf5026fb980ca8a902e69 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Tue, 19 Dec 2023 12:55:41 +0530 Subject: [PATCH 184/237] docs: consistency --- frappe/__init__.py | 5 +++-- frappe/build.py | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 1c228fc2d3..3d31adbcfa 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1003,7 +1003,8 @@ def has_permission( parent_doctype=None, ): """ - Return True if the user has permission `ptype` for given `doctype` or `doc` + Return True if the user has permission `ptype` for given `doctype` or `doc`. + Raise `frappe.PermissionError` if user isn't permitted and `throw` is truthy :param doctype: DocType for which permission is to be check. @@ -1152,7 +1153,7 @@ def set_value(doctype, docname, fieldname, value=None): def get_cached_doc(*args, **kwargs) -> "Document": - """Like `frappe.get_doc`, but return from cache if available.""" + """Identical to `frappe.get_doc`, but return from cache if available.""" if (key := can_cache_doc(args)) and (doc := cache.get_value(key)): return doc diff --git a/frappe/build.py b/frappe/build.py index af9bfce72d..03b830f0cb 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -133,9 +133,8 @@ def setup_assets(assets_archive): return directories_created -def download_frappe_assets(verbose=True): - """Downloads and sets up Frappe assets if they exist based on the current - commit HEAD. +def download_frappe_assets(verbose=True) -> bool: + """Download and set up Frappe assets if they exist based on the current commit HEAD. Return True if correctly setup else return False. """ frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD") From 2f277c66ae963f1c9e1d0d1393760425ec1c634b Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Tue, 19 Dec 2023 13:57:35 +0530 Subject: [PATCH 185/237] fix: use optional --- frappe/www/printview.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 79cb500609..df0ea1a064 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -93,7 +93,7 @@ def get_context(context) -> PrintContext: } -def get_print_format_doc(print_format_name: str, meta: "Meta") -> "PrintFormat" | None: +def get_print_format_doc(print_format_name: str, meta: "Meta") -> Optional["PrintFormat"]: """Return print format document.""" if not print_format_name: print_format_name = frappe.form_dict.format or meta.default_print_format or "Standard" @@ -110,7 +110,7 @@ def get_print_format_doc(print_format_name: str, meta: "Meta") -> "PrintFormat" def get_rendered_template( doc: "Document", - print_format: "PrintFormat" | None = None, + print_format: Optional["PrintFormat"] = None, meta: "Meta" = None, no_letterhead: bool | None = None, letterhead: str | None = None, From bb68d879c2fc21cecd48623cf7d881052d377b86 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Tue, 19 Dec 2023 14:01:41 +0530 Subject: [PATCH 186/237] fix: type hint issue --- frappe/www/printview.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index df0ea1a064..acff769e8d 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -388,9 +388,7 @@ def validate_key(key: str, doc: "Document") -> None: raise frappe.exceptions.InvalidKeyError -def get_letter_head( - doc: "Document", no_letterhead: bool, letterhead: str | None = None -) -> dict | "frappe._dict": +def get_letter_head(doc: "Document", no_letterhead: bool, letterhead: str | None = None) -> dict: if no_letterhead: return {} if letterhead: From e38b5e04f8ff4452f31e0f9e78501a4ab75f71cb Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Tue, 19 Dec 2023 15:46:21 +0530 Subject: [PATCH 187/237] fix(test_customize_form): drop checking of `flags.update_db` The value changed is being checked directly Checking the flag breaks if this or a similar test is run multiple times on the same database Also extract 255 into a separate variable Signed-off-by: Akhil Narang --- .../custom/doctype/customize_form/test_customize_form.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index 8a62d331be..7354c55efa 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -240,8 +240,9 @@ class TestCustomizeForm(FrappeTestCase): # Using Notification Log doctype as it doesn't have any other custom fields d = self.get_customize_form("Notification Log") + new_document_length = 255 document_name = d.get("fields", {"fieldname": "document_name"})[0] - document_name.length = 255 + document_name.length = new_document_length d.run_method("save_customization") self.assertEqual( @@ -250,11 +251,9 @@ class TestCustomizeForm(FrappeTestCase): {"doc_type": "Notification Log", "property": "length", "field_name": "document_name"}, "value", ), - "255", + str(new_document_length), ) - self.assertTrue(d.flags.update_db) - length = frappe.db.sql( """SELECT character_maximum_length FROM information_schema.columns @@ -262,7 +261,7 @@ class TestCustomizeForm(FrappeTestCase): AND column_name = 'document_name'""" )[0][0] - self.assertEqual(length, 255) + self.assertEqual(length, new_document_length) def test_custom_link(self): try: From 5deabdde2110937d57e5b606e21efeecdc789415 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 20 Dec 2023 11:56:36 +0530 Subject: [PATCH 188/237] fix: skip virtual fields in perm level checks during DB Query DB Query can't access virtual fields so it should ignore all virtual fields. --- frappe/model/__init__.py | 7 ++++++- frappe/model/db_query.py | 1 + frappe/model/meta.py | 13 +++++++++++-- frappe/tests/test_db_query.py | 8 ++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 9c0282a24d..ad29e31ee4 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -194,6 +194,8 @@ def get_permitted_fields( parenttype: str | None = None, user: str | None = None, permission_type: str | None = None, + *, + ignore_virtual=False, ) -> list[str]: meta = frappe.get_meta(doctype) valid_columns = meta.get_valid_columns() @@ -209,7 +211,10 @@ def get_permitted_fields( permission_type = "select" if frappe.only_has_select_perm(doctype, user=user) else "read" if permitted_fields := meta.get_permitted_fieldnames( - parenttype=parenttype, user=user, permission_type=permission_type + parenttype=parenttype, + user=user, + permission_type=permission_type, + with_virtual_fields=not ignore_virtual, ): if permission_type == "select": return permitted_fields diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 70ecedd3c8..a20053d646 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -635,6 +635,7 @@ class DatabaseQuery: doctype=self.doctype, parenttype=self.parent_doctype, permission_type=self.permission_map.get(self.doctype), + ignore_virtual=True, ) for i, field in enumerate(self.fields): diff --git a/frappe/model/meta.py b/frappe/model/meta.py index df04dc1eda..ae4e8ea5ef 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -565,7 +565,14 @@ class Meta(Document): self.high_permlevel_fields = [df for df in self.fields if df.permlevel > 0] return self.high_permlevel_fields - def get_permitted_fieldnames(self, parenttype=None, *, user=None, permission_type="read"): + def get_permitted_fieldnames( + self, + parenttype=None, + *, + user=None, + permission_type="read", + with_virtual_fields=True, + ): """Build list of `fieldname` with read perm level and all the higher perm levels defined. Note: If permissions are not defined for DocType, return all the fields with value. @@ -590,7 +597,9 @@ class Meta(Document): permitted_fieldnames.extend( df.fieldname - for df in self.get_fieldnames_with_value(with_field_meta=True, with_virtual_fields=True) + for df in self.get_fieldnames_with_value( + with_field_meta=True, with_virtual_fields=with_virtual_fields + ) if df.permlevel in permlevel_access ) return permitted_fieldnames diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 1ef4a23f65..87f044ff39 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -753,6 +753,14 @@ class TestDBQuery(FrappeTestCase): limit=50, ) + def test_virtual_field_get_list(self): + try: + frappe.get_list("Prepared Report", ["*"]) + frappe.get_list("Scheduled Job Type", ["*"]) + except Exception as e: + print(frappe.get_traceback()) + self.fail("get_list not working with virtual field") + def test_pluck_name(self): names = DatabaseQuery("DocType").execute(filters={"name": "DocType"}, pluck="name") self.assertEqual(names, ["DocType"]) From 1cbcf537d9a3c3b767158bdca73817bf2092529d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 20 Dec 2023 12:09:15 +0530 Subject: [PATCH 189/237] fix: Allow periodically clearing deleted documents --- frappe/core/doctype/deleted_document/deleted_document.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index aa6239c279..c99b6ad507 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -27,6 +27,14 @@ class DeletedDocument(Document): # end: auto-generated types pass + @staticmethod + def clear_old_logs(days=180): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Deleted Document") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + @frappe.whitelist() def restore(name, alert=True): From 3524cae48e00d9a9284f4550e2f3d9158cf4d838 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 20 Dec 2023 12:24:41 +0530 Subject: [PATCH 190/237] fix: ignore and gracefully handle img optimization failure PIL doesn't handle ALL image types. E.g. HEIC fails with bad error. --- frappe/utils/image.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/frappe/utils/image.py b/frappe/utils/image.py index eb57fbe0a7..71adb3c5c0 100644 --- a/frappe/utils/image.py +++ b/frappe/utils/image.py @@ -5,6 +5,8 @@ import os from PIL import Image +import frappe + def resize_images(path, maxdim=700): size = (maxdim, maxdim) @@ -51,22 +53,25 @@ def optimize_image( if content_type == "image/svg+xml": return content - image = Image.open(io.BytesIO(content)) - width, height = image.size - max_height = max(min(max_height, height * 0.8), 200) - max_width = max(min(max_width, width * 0.8), 200) - image_format = content_type.split("/")[1] - size = max_width, max_height - image.thumbnail(size, Image.Resampling.LANCZOS) + try: + image = Image.open(io.BytesIO(content)) + width, height = image.size + max_height = max(min(max_height, height * 0.8), 200) + max_width = max(min(max_width, width * 0.8), 200) + image_format = content_type.split("/")[1] + size = max_width, max_height + image.thumbnail(size, Image.Resampling.LANCZOS) - output = io.BytesIO() - image.save( - output, - format=image_format, - optimize=optimize, - quality=quality, - save_all=True if image_format == "gif" else None, - ) - - optimized_content = output.getvalue() - return optimized_content if len(optimized_content) < len(content) else content + output = io.BytesIO() + image.save( + output, + format=image_format, + optimize=optimize, + quality=quality, + save_all=True if image_format == "gif" else None, + ) + optimized_content = output.getvalue() + return optimized_content if len(optimized_content) < len(content) else content + except Exception as e: + frappe.msgprint(frappe._("Failed to optimize image: {0}").format(str(e))) + return content From 39359f9793ff533e7ec37174ff82f0cdaf049be0 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Wed, 20 Dec 2023 12:43:42 +0530 Subject: [PATCH 191/237] fix(sentry): `name` field is ignored Only `id`, `username`, `email`, and `ip_address` are accepted here Signed-off-by: Akhil Narang --- frappe/utils/sentry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/utils/sentry.py b/frappe/utils/sentry.py index c7843d8c10..8ca66a15d7 100644 --- a/frappe/utils/sentry.py +++ b/frappe/utils/sentry.py @@ -38,7 +38,8 @@ def capture_exception(message: str | None = None) -> None: evt_processor = _make_wsgi_event_processor(frappe.request.environ, False) scope.add_event_processor(evt_processor) scope.set_tag("site", frappe.local.site) - scope.set_user({"name": getattr(frappe.session, "user", "Unidentified")}) + user = getattr(frappe.session, "user", "Unidentified") + scope.set_user({"id": user, "email": user}) # Extract `X-Frappe-Request-ID` to store as a separate field if its present if trace_id := frappe.monitor.get_trace_id(): From b51a479fc21a98c456b5b06dc4b2bf04de24ed18 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 20 Dec 2023 12:59:58 +0530 Subject: [PATCH 192/237] fix: don't allow deleting original doc if amendment exists --- frappe/model/delete_doc.py | 2 +- frappe/tests/test_linked_with.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index e2202882b1..b6ab2546bf 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -254,7 +254,7 @@ def check_if_doc_is_linked(doc, method="Delete"): for lf in link_fields: link_dt, link_field, issingle = lf["parent"], lf["fieldname"], lf["issingle"] - if link_dt in ignored_doctypes or link_field == "amended_from": + if link_dt in ignored_doctypes or (link_field == "amended_from" and method == "Cancel"): continue try: diff --git a/frappe/tests/test_linked_with.py b/frappe/tests/test_linked_with.py index 70dddba334..2ba4f04c3f 100644 --- a/frappe/tests/test_linked_with.py +++ b/frappe/tests/test_linked_with.py @@ -61,6 +61,7 @@ class TestLinkedWith(FrappeTestCase): def tearDown(self): for doctype in ["Parent DocType", "Child DocType1", "Child DocType2"]: frappe.delete_doc("DocType", doctype) + frappe.db.commit() def test_get_doctype_references_by_link_field(self): references = linked_with.get_references_across_doctypes_by_link_field( @@ -139,3 +140,17 @@ class TestLinkedWith(FrappeTestCase): child_record.cancel() child_record.delete() parent_record.delete() + + def test_check_delete_integrity(self): + """Don't allow deleting cancelled document if amendment exists""" + doc = frappe.get_doc({"doctype": "Parent DocType"}).insert() + doc.submit() + doc.cancel() + + amendment = frappe.copy_doc(doc) + amendment.amended_from = doc.name + amendment.docstatus = 0 + amendment.insert() + amendment.submit() + + self.assertRaises(frappe.LinkExistsError, doc.delete) From 02f3fd8470751cbabec49ef0c1eb7545224e6d9b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 20 Dec 2023 13:57:48 +0530 Subject: [PATCH 193/237] test: retry flaky password_strength test thrice --- frappe/tests/test_password_strength.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/tests/test_password_strength.py b/frappe/tests/test_password_strength.py index 5dc87d185d..4296548861 100644 --- a/frappe/tests/test_password_strength.py +++ b/frappe/tests/test_password_strength.py @@ -3,10 +3,18 @@ from string import printable from time import time from unittest import TestCase +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed + from frappe.utils.password_strength import test_password_strength class TestPasswordStrength(TestCase): + @retry( + retry=retry_if_exception_type(AssertionError), + stop=stop_after_attempt(3), + wait=wait_fixed(0.5), + reraise=True, + ) def test_long_password(self): password = "".join(random.choice(printable) for _ in range(600)) From ecdd7b3e374c8e76d417df9897edc00750e1141b Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Wed, 20 Dec 2023 14:02:32 +0530 Subject: [PATCH 194/237] docs: consistency --- frappe/core/doctype/server_script/server_script.py | 2 +- frappe/core/doctype/user/user.py | 4 ++-- frappe/database/operator_map.py | 14 +++++++------- frappe/utils/data.py | 2 +- frappe/utils/identicon.py | 2 +- frappe/utils/scheduler.py | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index eaa8bc4b96..a9e047d9b2 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -135,7 +135,7 @@ class ServerScript(Document): frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user. Return: - dict: Evaluate self.script with frappe.utils.safe_exec.safe_exec and return the flags set in it's safe globals. + dict: Evaluate self.script with frappe.utils.safe_exec.safe_exec and return the flags set in its safe globals. """ if self.enable_rate_limit: diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index a56bfaaca0..1a13a20e4e 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -1083,7 +1083,7 @@ def user_query(doctype, txt, searchfield, start, page_len, filters): def get_total_users(): - """Return total no. of system users.""" + """Return total number of system users.""" return flt( frappe.db.sql( """SELECT SUM(`simultaneous_sessions`) @@ -1131,7 +1131,7 @@ def get_active_users(): def get_website_users(): - """Return total no. of website users.""" + """Return total number of website users.""" return frappe.db.count("User", filters={"enabled": True, "user_type": "Website User"}) diff --git a/frappe/database/operator_map.py b/frappe/database/operator_map.py index 8f77db5687..72ed8b4a75 100644 --- a/frappe/database/operator_map.py +++ b/frappe/database/operator_map.py @@ -18,7 +18,7 @@ def like(key: Field, value: str) -> frappe.qb: value (str): criterion Return: - frappe.qb: `frappe.qb object with `LIKE` + frappe.qb: `frappe.qb` object with `LIKE` """ return key.like(value) @@ -31,7 +31,7 @@ def func_in(key: Field, value: list | tuple) -> frappe.qb: value (Union[int, str]): criterion Return: - frappe.qb: `frappe.qb object with `IN` + frappe.qb: `frappe.qb` object with `IN` """ if isinstance(value, str): value = value.split(",") @@ -46,7 +46,7 @@ def not_like(key: Field, value: str) -> frappe.qb: value (str): criterion Return: - frappe.qb: `frappe.qb object with `NOT LIKE` + frappe.qb: `frappe.qb` object with `NOT LIKE` """ return key.not_like(value) @@ -59,7 +59,7 @@ def func_not_in(key: Field, value: list | tuple | str): value (Union[int, str]): criterion Return: - frappe.qb: `frappe.qb object with `NOT IN` + frappe.qb: `frappe.qb` object with `NOT IN` """ if isinstance(value, str): value = value.split(",") @@ -74,7 +74,7 @@ def func_regex(key: Field, value: str) -> frappe.qb: value (str): criterion Return: - frappe.qb: `frappe.qb object with `REGEX` + frappe.qb: `frappe.qb` object with `REGEX` """ return key.regex(value) @@ -87,7 +87,7 @@ def func_between(key: Field, value: list | tuple) -> frappe.qb: value (Union[int, str]): criterion Return: - frappe.qb: `frappe.qb object with `BETWEEN` + frappe.qb: `frappe.qb` object with `BETWEEN` """ return key[slice(*value)] @@ -105,7 +105,7 @@ def func_timespan(key: Field, value: str) -> frappe.qb: value (str): criterion Return: - frappe.qb: `frappe.qb object with `TIMESPAN` + frappe.qb: `frappe.qb` object with `TIMESPAN` """ return func_between(key, get_timespan_date_range(value)) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 016c62ba79..8a40235ea2 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -126,7 +126,7 @@ def getdate( def get_datetime( - datetime_str: Optional["DateTimeLikeObject"] = None, + datetime_str: "DateTimeLikeObject" | list | tuple | datetime.timedelta | None = None, ) -> datetime.datetime | None: """Return the below mentioned values based on the given `datetime_str`: diff --git a/frappe/utils/identicon.py b/frappe/utils/identicon.py index 43517087d0..724af20464 100644 --- a/frappe/utils/identicon.py +++ b/frappe/utils/identicon.py @@ -28,7 +28,7 @@ class Identicon: self.hash = self.digest(str_) def digest(self, str_): - """Return a md5 numeric hash.""" + """Return an MD5 numeric hash.""" return int(md5(str_.encode("utf-8"), usedforsecurity=False).hexdigest(), 16) def calculate(self): diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index 8c048c1a16..88cb85b667 100755 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -141,8 +141,8 @@ def disable_scheduler(): def schedule_jobs_based_on_activity(check_time=None): - """Return True for active sites defined by Activity Log. - Also return True for inactive sites once in 24 hours.""" + """Return True for active sites as defined by `Activity Log`. + Also return True for inactive sites once every 24 hours based on `Scheduled Job Log`.""" if is_dormant(check_time=check_time): # ensure last job is one day old last_job_timestamp = _get_last_modified_timestamp("Scheduled Job Log") From 369844a9cbd436439b45d5036beed73c26ea9929 Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Wed, 20 Dec 2023 14:09:01 +0530 Subject: [PATCH 195/237] fix: type hint --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 8a40235ea2..bd4dc894fe 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -126,7 +126,7 @@ def getdate( def get_datetime( - datetime_str: "DateTimeLikeObject" | list | tuple | datetime.timedelta | None = None, + datetime_str: Optional["DateTimeLikeObject"] | tuple | list = None, ) -> datetime.datetime | None: """Return the below mentioned values based on the given `datetime_str`: From 5213915d6602f7f80c462c76d22666122fd9185a Mon Sep 17 00:00:00 2001 From: Daizy Modi Date: Wed, 20 Dec 2023 14:11:01 +0530 Subject: [PATCH 196/237] fix: render new doctype dialog box if doctype is not copied (#23854) --- frappe/core/doctype/doctype/doctype.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index c21654a109..ebc4d85d0b 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -3,7 +3,7 @@ frappe.ui.form.on("DocType", { onload: function (frm) { - if (frm.is_new()) { + if (frm.is_new() && !frm.doc?.fields) { frappe.listview_settings["DocType"].new_doctype_dialog(); } }, From b4d3132bf421963f20d733fc426d86143d7d834f Mon Sep 17 00:00:00 2001 From: Kunhi Date: Wed, 20 Dec 2023 14:56:17 +0400 Subject: [PATCH 197/237] fix: optimize clear_permissions_cache method (#23858) * fix:optimize clear_permissions_cache method * fix: faster clear user cache on perm change --------- Co-authored-by: Ankush Menat --- frappe/core/doctype/doctype/doctype.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 83b5395a4e..8e1ab0512c 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -1668,22 +1668,12 @@ def validate_permissions_for_doctype(doctype, for_remove=False, alert=False): def clear_permissions_cache(doctype): + from frappe.cache_manager import clear_user_cache + frappe.clear_cache(doctype=doctype) delete_notification_count_for(doctype) - for user in frappe.db.sql_list( - """ - SELECT - DISTINCT `tabHas Role`.`parent` - FROM - `tabHas Role`, - `tabDocPerm` - WHERE `tabDocPerm`.`parent` = %s - AND `tabDocPerm`.`role` = `tabHas Role`.`role` - AND `tabHas Role`.`parenttype` = 'User' - """, - doctype, - ): - frappe.clear_cache(user=user) + + clear_user_cache() def validate_permissions(doctype, for_remove=False, alert=False): From 3015852ce14a76be432b338e1ea28fdb46d42a20 Mon Sep 17 00:00:00 2001 From: bourouffala Date: Wed, 20 Dec 2023 12:13:20 +0100 Subject: [PATCH 198/237] fix: Error when displaying dashboard with number card using average and sum functions (#23883) --- frappe/model/db_query.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index a20053d646..c49ef57129 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -1080,6 +1080,8 @@ class DatabaseQuery: self.fields[0].lower().startswith("count(") or self.fields[0].lower().startswith("min(") or self.fields[0].lower().startswith("max(") + or self.fields[0].lower().startswith("sum(") + or self.fields[0].lower().startswith("avg(") ) and not self.group_by ) From 91dce0b00799f48b421fc4a15a9e93bc7cf94b1a Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Wed, 20 Dec 2023 17:34:43 +0530 Subject: [PATCH 199/237] feat(sentry): drop events which have `frappe.throw` (#23886) Signed-off-by: Akhil Narang --- frappe/public/js/sentry.bundle.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frappe/public/js/sentry.bundle.js b/frappe/public/js/sentry.bundle.js index edef9bc363..9757595af4 100644 --- a/frappe/public/js/sentry.bundle.js +++ b/frappe/public/js/sentry.bundle.js @@ -9,4 +9,15 @@ Sentry.init({ user: { id: frappe.boot.user.name ?? "Unidentified" }, tags: { site: frappe.boot.sitename }, }, + beforeSend(event, hint) { + // Check if it was caused by frappe.throw() + if ( + hint.originalException instanceof Error && + hint.originalException.stack && + hint.originalException.stack.includes("frappe.throw") + ) { + return null; + } + return event; + }, }); From 44667901bf2f351c4dfd699b0ae95313e5bef969 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Wed, 13 Dec 2023 12:23:47 +0530 Subject: [PATCH 200/237] feat: allow setting a custom http status code for redirects Signed-off-by: Akhil Narang --- frappe/exceptions.py | 3 ++- .../website_route_redirect.json | 18 +++++++++++++++--- .../website_route_redirect.py | 2 ++ frappe/website/page_renderers/redirect_page.py | 2 +- frappe/website/path_resolver.py | 12 +++++++----- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/frappe/exceptions.py b/frappe/exceptions.py index f4bcb661f1..2258c6e5ae 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -64,7 +64,8 @@ class RequestToken(Exception): class Redirect(Exception): - http_status_code = 301 + def __init__(self, http_status_code: int = 301): + self.http_status_code = http_status_code class CSRFTokenError(Exception): diff --git a/frappe/website/doctype/website_route_redirect/website_route_redirect.json b/frappe/website/doctype/website_route_redirect/website_route_redirect.json index d561a1be10..30ace0b8ba 100644 --- a/frappe/website/doctype/website_route_redirect/website_route_redirect.json +++ b/frappe/website/doctype/website_route_redirect/website_route_redirect.json @@ -1,10 +1,12 @@ { + "actions": [], "creation": "2019-05-07 11:08:35.889625", "doctype": "DocType", "engine": "InnoDB", "field_order": [ "source", - "target" + "target", + "redirect_http_status" ], "fields": [ { @@ -20,10 +22,19 @@ "in_list_view": 1, "label": "Target", "reqd": 1 + }, + { + "default": "301", + "fieldname": "redirect_http_status", + "fieldtype": "Int", + "label": "Redirect HTTP Status", + "options": "301\n302\n307\n308", + "reqd": 1 } ], "istable": 1, - "modified": "2019-05-07 11:11:46.867684", + "links": [], + "modified": "2023-12-13 12:09:50.726082", "modified_by": "Administrator", "module": "Website", "name": "Website Route Redirect", @@ -31,5 +42,6 @@ "permissions": [], "quick_entry": 1, "sort_field": "modified", - "sort_order": "ASC" + "sort_order": "ASC", + "states": [] } \ No newline at end of file diff --git a/frappe/website/doctype/website_route_redirect/website_route_redirect.py b/frappe/website/doctype/website_route_redirect/website_route_redirect.py index 32949fb229..616c8f28ac 100644 --- a/frappe/website/doctype/website_route_redirect/website_route_redirect.py +++ b/frappe/website/doctype/website_route_redirect/website_route_redirect.py @@ -17,7 +17,9 @@ class WebsiteRouteRedirect(Document): parent: DF.Data parentfield: DF.Data parenttype: DF.Data + redirect_http_status: DF.Int source: DF.SmallText target: DF.SmallText # end: auto-generated types + pass diff --git a/frappe/website/page_renderers/redirect_page.py b/frappe/website/page_renderers/redirect_page.py index be4a7bdf45..cfb9a085d8 100644 --- a/frappe/website/page_renderers/redirect_page.py +++ b/frappe/website/page_renderers/redirect_page.py @@ -14,7 +14,7 @@ class RedirectPage: return build_response( self.path, "", - 301, + self.http_status_code, { "Location": frappe.flags.redirect_location or (frappe.local.response or {}).get("location"), "Cache-Control": "no-store, no-cache, must-revalidate", diff --git a/frappe/website/path_resolver.py b/frappe/website/path_resolver.py index a91a51c0da..4837123d8a 100644 --- a/frappe/website/path_resolver.py +++ b/frappe/website/path_resolver.py @@ -35,8 +35,8 @@ class PathResolver: try: resolve_redirect(self.path, request.query_string) - except frappe.Redirect: - return frappe.flags.redirect_location, RedirectPage(self.path) + except frappe.Redirect as e: + return frappe.flags.redirect_location, RedirectPage(self.path, e.http_status_code) endpoint = resolve_path(self.path) @@ -105,7 +105,9 @@ def resolve_redirect(path, query_string=None): ] """ redirects = frappe.get_hooks("website_redirects") - redirects += frappe.get_all("Website Route Redirect", ["source", "target"], order_by=None) + redirects += frappe.get_all( + "Website Route Redirect", ["source", "target", "redirect_http_status"], order_by=None + ) if not redirects: return @@ -124,14 +126,14 @@ def resolve_redirect(path, query_string=None): try: match = re.match(pattern, path_to_match) - except re.error as e: + except re.error: frappe.log_error("Broken Redirect: " + pattern) if match: redirect_to = re.sub(pattern, rule["target"], path_to_match) frappe.flags.redirect_location = redirect_to frappe.cache.hset("website_redirects", path_to_match, redirect_to) - raise frappe.Redirect + raise frappe.Redirect(rule.get("redirect_http_status", 301)) def resolve_path(path): From c8816a8b6a66fb0a7d0643474c1999a0971c3d38 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Wed, 13 Dec 2023 12:32:35 +0530 Subject: [PATCH 201/237] fix(tests): fix redirect tests by adding new field + add another test scenario Signed-off-by: Akhil Narang --- frappe/tests/test_website.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index b7b33b3531..313a1e5dfa 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -170,10 +170,18 @@ class TestWebsite(FrappeTestCase): dict( source=r"/courses/course\?course=(.*)", target=r"/courses/\1", match_with_query_string=True ), + dict( + source="/test307", + target="/test", + redirect_http_status=307, + ), ] website_settings = frappe.get_doc("Website Settings") - website_settings.append("route_redirects", {"source": "/testsource", "target": "/testtarget"}) + website_settings.append( + "route_redirects", + {"source": "/testsource", "target": "/testtarget", "redirect_http_status": 301}, + ) website_settings.save() set_request(method="GET", path="/testfrom") @@ -205,6 +213,11 @@ class TestWebsite(FrappeTestCase): self.assertEqual(response.status_code, 301) self.assertEqual(response.headers.get("Location"), "/courses/data") + set_request(method="GET", path="/test307") + response = get_response() + self.assertEqual(response.status_code, 307) + self.assertEqual(response.headers.get("Location"), "/test") + delattr(frappe.hooks, "website_redirects") frappe.cache.delete_key("app_hooks") From 6819a38fbe11aaa0fd5d4cf1b93e11e848efe685 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 18 Dec 2023 16:11:08 +0530 Subject: [PATCH 202/237] fix: set correct response code for redirections Signed-off-by: Akhil Narang --- frappe/website/serve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/website/serve.py b/frappe/website/serve.py index acae44940e..2261683406 100644 --- a/frappe/website/serve.py +++ b/frappe/website/serve.py @@ -16,8 +16,8 @@ def get_response(path=None, http_status_code=200): path_resolver = PathResolver(path, http_status_code) endpoint, renderer_instance = path_resolver.resolve() response = renderer_instance.render() - except frappe.Redirect: - return RedirectPage(endpoint or path, http_status_code).render() + except frappe.Redirect as e: + return RedirectPage(endpoint or path, e.http_status_code).render() except frappe.PermissionError as e: response = NotPermittedPage(endpoint, http_status_code, exception=e).render() except frappe.PageDoesNotExistError: From 2224d86c9487587606d5894a642fd7b06511fd9e Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Wed, 20 Dec 2023 17:47:16 +0530 Subject: [PATCH 203/237] refactor: cache status code as well Signed-off-by: Akhil Narang --- frappe/tests/test_website.py | 5 +++++ frappe/website/path_resolver.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index 313a1e5dfa..2ab4ee8a0e 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -218,6 +218,11 @@ class TestWebsite(FrappeTestCase): self.assertEqual(response.status_code, 307) self.assertEqual(response.headers.get("Location"), "/test") + set_request(method="POST", path="/test307") + response = get_response() + self.assertEqual(response.status_code, 307) + self.assertEqual(response.headers.get("Location"), "/test") + delattr(frappe.hooks, "website_redirects") frappe.cache.delete_key("app_hooks") diff --git a/frappe/website/path_resolver.py b/frappe/website/path_resolver.py index 4837123d8a..11da96b275 100644 --- a/frappe/website/path_resolver.py +++ b/frappe/website/path_resolver.py @@ -115,6 +115,9 @@ def resolve_redirect(path, query_string=None): redirect_to = frappe.cache.hget("website_redirects", path) if redirect_to: + if isinstance(redirect_to, dict): + frappe.flags.redirect_location = redirect_to["path"] + raise frappe.Redirect(redirect_to["status_code"]) frappe.flags.redirect_location = redirect_to raise frappe.Redirect @@ -132,8 +135,11 @@ def resolve_redirect(path, query_string=None): if match: redirect_to = re.sub(pattern, rule["target"], path_to_match) frappe.flags.redirect_location = redirect_to - frappe.cache.hset("website_redirects", path_to_match, redirect_to) - raise frappe.Redirect(rule.get("redirect_http_status", 301)) + status_code = rule.get("redirect_http_status", 301) + frappe.cache.hset( + "website_redirects", path_to_match, {"path": redirect_to, "status_code": status_code} + ) + raise frappe.Redirect(status_code) def resolve_path(path): From 3d0ea8cdff98607ce402d69ae801639040994043 Mon Sep 17 00:00:00 2001 From: Justine Jay Date: Thu, 21 Dec 2023 06:56:41 +0800 Subject: [PATCH 204/237] fix: add skip_dirty_trigger param for child_table --- frappe/public/js/frappe/form/form.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 044af703e1..4f42b3ecaa 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -346,13 +346,20 @@ frappe.ui.form.Form = class FrappeForm { // using $.each to preserve df via closure $.each(table_fields, function (i, df) { - frappe.model.on(df.options, "*", function (fieldname, value, doc) { - if (doc.parent == me.docname && doc.parentfield === df.fieldname) { - me.dirty(); - me.fields_dict[df.fieldname].grid.set_value(fieldname, value, doc); - return me.script_manager.trigger(fieldname, doc.doctype, doc.name); + frappe.model.on( + df.options, + "*", + function (fieldname, value, doc, skip_dirty_trigger = false) { + if (doc.parent == me.docname && doc.parentfield === df.fieldname) { + if (!skip_dirty_trigger) { + me.dirty(); + } + + me.fields_dict[df.fieldname].grid.set_value(fieldname, value, doc); + return me.script_manager.trigger(fieldname, doc.doctype, doc.name); + } } - }); + ); }); } From 25c75cacc2f6de872ec13da0c8dc0272a93c234c Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 21 Dec 2023 12:12:38 +0530 Subject: [PATCH 205/237] fix: allow aspect ratio options to crop from sidebar image field --- frappe/public/js/frappe/form/controls/attach.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/attach.js b/frappe/public/js/frappe/form/controls/attach.js index e7949ad9e2..992805025a 100644 --- a/frappe/public/js/frappe/form/controls/attach.js +++ b/frappe/public/js/frappe/form/controls/attach.js @@ -63,7 +63,6 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro on_attach_doc_image() { this.set_upload_options(); this.upload_options.restrictions.allowed_file_types = ["image/*"]; - this.upload_options.restrictions.crop_image_aspect_ratio = 1; this.file_uploader = new frappe.ui.FileUploader(this.upload_options); } set_upload_options() { From 2fc3e2cddd617e1cf8fb5feac37bbee54bc500d6 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 21 Dec 2023 12:13:06 +0530 Subject: [PATCH 206/237] chore: close sidebar was failing --- frappe/public/js/frappe/form/sidebar/user_image.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/sidebar/user_image.js b/frappe/public/js/frappe/form/sidebar/user_image.js index ae6167e184..b022b157b9 100644 --- a/frappe/public/js/frappe/form/sidebar/user_image.js +++ b/frappe/public/js/frappe/form/sidebar/user_image.js @@ -74,7 +74,7 @@ frappe.ui.form.setup_user_image_event = function (frm) { } field.$input.trigger("attach_doc_image"); // close sidebar - frm.page.close_sidebar(); + frm.page.close_sidebar?.(); } else { /// on remove event for a sidebar image wrapper remove attach file. frm.attachments.remove_attachment_by_filename( From cf69a6e34084e64c8add4020ed1418fef559a083 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 21 Dec 2023 14:20:31 +0530 Subject: [PATCH 207/237] build(deps): bump datatable to latest --- frappe/public/js/frappe/views/reports/query_report.js | 4 ++-- package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 3bda8340b8..caad1f99be 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1259,13 +1259,13 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { width: parseInt(column.width) || null, editable: false, compareValue: compareFn, - format: (value, row, column, data, filter) => { + format: (value, row, column, data, filter, data1) => { if (this.report_settings.formatter) { return this.report_settings.formatter( value, row, column, - data, + data1, format_cell, filter ); diff --git a/package.json b/package.json index cfde368887..5ec8402fe3 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "fast-deep-equal": "^2.0.1", "fast-glob": "^3.2.5", "frappe-charts": "2.0.0-rc22", - "frappe-datatable": "^1.17.5", + "frappe-datatable": "^1.17.9", "frappe-gantt": "^0.6.0", "highlight.js": "^10.4.1", "html5-qrcode": "^2.3.8", diff --git a/yarn.lock b/yarn.lock index f6045227c5..98a9877253 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1497,10 +1497,10 @@ frappe-charts@2.0.0-rc22: resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc22.tgz#9a5a747febdc381a1d4d7af96e89cf519dfba8c0" integrity sha512-N7f/8979wJCKjusOinaUYfMxB80YnfuVLrSkjpj4LtyqS0BGS6SuJxUnb7Jl4RWUFEIs7zEhideIKnyLeFZF4Q== -frappe-datatable@^1.17.5: - version "1.17.5" - resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.17.5.tgz#342814d7f9eb502f55639c1b09d44d9eca782f78" - integrity sha512-FJdpsj/xACuk553FXMMqvtTQZII9P9TEtAwOiaKN+AwDAjfCO4b5vhSTmdNp2Kgf26bnZU9QVfRvh2gDtpj3OA== +frappe-datatable@^1.17.9: + version "1.17.9" + resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.17.9.tgz#5ef4e5d335079ab5bf2abfecc916e31ecf17a5cb" + integrity sha512-C1U5YKk7kP32eiHVnv1AdY5LafKKoGrcDpbErqM95PYrhanaq2Uvkvdsjo6yioLpPfnvFD8Vihm4JoGc8FjDcw== dependencies: hyperlist "^1.0.0-beta" lodash "^4.17.5" From 5e2ace4c086d4061ab6e27585023430caf39008b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 21 Dec 2023 23:19:06 +0530 Subject: [PATCH 208/237] perf: Use set for tracking whitelisted methods (#23905) List look ups are O(N). This is still a microoptimization at best considering other overhead. --- frappe/__init__.py | 12 ++++++------ frappe/desk/treeview.py | 3 +-- frappe/model/mapper.py | 6 ++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 3d31adbcfa..158d5c9fa4 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -780,9 +780,9 @@ def sendmail( return builder.process(send_now=now) -whitelisted = [] -guest_methods = [] -xss_safe_methods = [] +whitelisted = set() +guest_methods = set() +xss_safe_methods = set() allowed_http_methods_for_whitelisted_func = {} @@ -821,14 +821,14 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None): else: fn = validate_argument_types(fn, apply_condition=in_request_or_test) - whitelisted.append(fn) + whitelisted.add(fn) allowed_http_methods_for_whitelisted_func[fn] = methods if allow_guest: - guest_methods.append(fn) + guest_methods.add(fn) if xss_safe: - xss_safe_methods.append(fn) + xss_safe_methods.add(fn) return method or fn diff --git a/frappe/desk/treeview.py b/frappe/desk/treeview.py index f8b2a67c82..dcce6f3850 100644 --- a/frappe/desk/treeview.py +++ b/frappe/desk/treeview.py @@ -15,8 +15,7 @@ def get_all_nodes(doctype, label, parent, tree_method, **filters): tree_method = frappe.get_attr(tree_method) - if tree_method not in frappe.whitelisted: - frappe.throw(_("Not Permitted"), frappe.PermissionError) + frappe.is_whitelisted(tree_method) data = tree_method(doctype, parent, **filters) out = [dict(parent=label, data=data)] diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index d1799682ed..78f329ee83 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -22,8 +22,7 @@ def make_mapped_doc(method, source_name, selected_children=None, args=None): method = frappe.get_attr(method) - if method not in frappe.whitelisted: - raise frappe.PermissionError + frappe.is_whitelisted(method) if selected_children: selected_children = json.loads(selected_children) @@ -46,8 +45,7 @@ def map_docs(method, source_names, target_doc, args=None): """ method = frappe.get_attr(method) - if method not in frappe.whitelisted: - raise frappe.PermissionError + frappe.is_whitelisted(method) for src in json.loads(source_names): _args = (src, target_doc, json.loads(args)) if args else (src, target_doc) From 1d23888475ff668500a0f7817a1310ae540e396e Mon Sep 17 00:00:00 2001 From: Corentin Flr <10946971+cogk@users.noreply.github.com> Date: Thu, 21 Dec 2023 18:50:25 +0100 Subject: [PATCH 209/237] fix(translations): Add context to Image Cropper text (#23908) --- frappe/public/js/frappe/file_uploader/ImageCropper.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/file_uploader/ImageCropper.vue b/frappe/public/js/frappe/file_uploader/ImageCropper.vue index f57418613c..6a0748c7b9 100644 --- a/frappe/public/js/frappe/file_uploader/ImageCropper.vue +++ b/frappe/public/js/frappe/file_uploader/ImageCropper.vue @@ -97,19 +97,19 @@ onMounted(() => { let aspect_ratio_buttons = computed(() => { return [ { - label: __("1:1"), + label: __("1:1", null, "Image Cropper"), value: 1, }, { - label: __("4:3"), + label: __("4:3", null, "Image Cropper"), value: 4 / 3, }, { - label: __("16:9"), + label: __("16:9", null, "Image Cropper"), value: 16 / 9, }, { - label: __("Free"), + label: __("Free", null, "Image Cropper"), value: NaN, }, ]; From ff4dca3e16125731b241e9f79c2db10be1891f0b Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Fri, 22 Dec 2023 12:22:48 +0530 Subject: [PATCH 210/237] fix(redirect): make the status codes a `select` field instead of `int` Drop mandatory, assume sane defaults The current implementation broke old users of redirects like helpdesk app Signed-off-by: Akhil Narang --- frappe/tests/test_website.py | 11 ++++++++++- .../website_route_redirect.json | 7 +++---- .../website_route_redirect/website_route_redirect.py | 2 +- frappe/website/path_resolver.py | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index 2ab4ee8a0e..4e768e5467 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -180,7 +180,11 @@ class TestWebsite(FrappeTestCase): website_settings = frappe.get_doc("Website Settings") website_settings.append( "route_redirects", - {"source": "/testsource", "target": "/testtarget", "redirect_http_status": 301}, + {"source": "/testsource", "target": "/testtarget"}, + ) + website_settings.append( + "route_redirects", + {"source": "/testdoc307", "target": "/testtarget", "redirect_http_status": 307}, ) website_settings.save() @@ -208,6 +212,11 @@ class TestWebsite(FrappeTestCase): self.assertEqual(response.status_code, 301) self.assertEqual(response.headers.get("Location"), "/testtarget") + set_request(method="GET", path="/testdoc307") + response = get_response() + self.assertEqual(response.status_code, 307) + self.assertEqual(response.headers.get("Location"), "/testtarget") + set_request(method="GET", path="/courses/course?course=data") response = get_response() self.assertEqual(response.status_code, 301) diff --git a/frappe/website/doctype/website_route_redirect/website_route_redirect.json b/frappe/website/doctype/website_route_redirect/website_route_redirect.json index 30ace0b8ba..d105f11053 100644 --- a/frappe/website/doctype/website_route_redirect/website_route_redirect.json +++ b/frappe/website/doctype/website_route_redirect/website_route_redirect.json @@ -26,15 +26,14 @@ { "default": "301", "fieldname": "redirect_http_status", - "fieldtype": "Int", + "fieldtype": "Select", "label": "Redirect HTTP Status", - "options": "301\n302\n307\n308", - "reqd": 1 + "options": "301\n302\n307\n308" } ], "istable": 1, "links": [], - "modified": "2023-12-13 12:09:50.726082", + "modified": "2023-12-22 12:21:28.436139", "modified_by": "Administrator", "module": "Website", "name": "Website Route Redirect", diff --git a/frappe/website/doctype/website_route_redirect/website_route_redirect.py b/frappe/website/doctype/website_route_redirect/website_route_redirect.py index 616c8f28ac..06f29117e2 100644 --- a/frappe/website/doctype/website_route_redirect/website_route_redirect.py +++ b/frappe/website/doctype/website_route_redirect/website_route_redirect.py @@ -17,7 +17,7 @@ class WebsiteRouteRedirect(Document): parent: DF.Data parentfield: DF.Data parenttype: DF.Data - redirect_http_status: DF.Int + redirect_http_status: DF.Literal["301", "302", "307", "308"] source: DF.SmallText target: DF.SmallText # end: auto-generated types diff --git a/frappe/website/path_resolver.py b/frappe/website/path_resolver.py index 0ff1a8f0e3..fefec2b4eb 100644 --- a/frappe/website/path_resolver.py +++ b/frappe/website/path_resolver.py @@ -135,7 +135,7 @@ def resolve_redirect(path, query_string=None): if match: redirect_to = re.sub(pattern, rule["target"], path_to_match) frappe.flags.redirect_location = redirect_to - status_code = rule.get("redirect_http_status", 301) + status_code = rule.get("redirect_http_status") or 301 frappe.cache.hset( "website_redirects", path_to_match, {"path": redirect_to, "status_code": status_code} ) From 4d68a46b10101355abc77ba52de1afae8ebac14e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 22 Dec 2023 13:32:36 +0530 Subject: [PATCH 211/237] fix: Set default child table fields on save (#23913) We set defaults on creation of new doc but when you append a new child doc on existing document it doesn't seem to set the defaults. This seems like a bug and not a deliberate choice. --- frappe/core/doctype/doctype/test_doctype.py | 2 ++ frappe/model/document.py | 9 ++++++--- frappe/tests/test_document.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index c1c7589564..a5657f590a 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -786,6 +786,7 @@ def new_doctype( depends_on: str = "", fields: list[dict] | None = None, custom: bool = True, + default: str | None = None, **kwargs, ): if not name: @@ -803,6 +804,7 @@ def new_doctype( "fieldname": "some_fieldname", "fieldtype": "Data", "unique": unique, + "default": default, "depends_on": depends_on, } ], diff --git a/frappe/model/document.py b/frappe/model/document.py index 9135b07dc5..80593275d2 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -356,6 +356,7 @@ class Document(BaseDocument): return self.insert() self.check_if_locked() + self._set_defaults() self.check_permission("write", "save") self.set_user_and_timestamp() @@ -771,8 +772,9 @@ class Document(BaseDocument): if frappe.flags.in_import: return - new_doc = frappe.new_doc(self.doctype, as_dict=True) - self.update_if_missing(new_doc) + if self.is_new(): + new_doc = frappe.new_doc(self.doctype, as_dict=True) + self.update_if_missing(new_doc) # children for df in self.meta.get_table_fields(): @@ -780,7 +782,8 @@ class Document(BaseDocument): value = self.get(df.fieldname) if isinstance(value, list): for d in value: - d.update_if_missing(new_doc) + if d.is_new(): + d.update_if_missing(new_doc) def check_if_latest(self): """Checks if `modified` timestamp provided by document being updated is same as the diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 9ee06c13bf..176dd169aa 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -6,6 +6,7 @@ from unittest.mock import Mock, patch import frappe from frappe.app import make_form_dict +from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.desk.doctype.note.note import Note from frappe.model.naming import make_autoname, parse_naming_series, revert_series_if_last from frappe.tests.utils import FrappeTestCase @@ -63,6 +64,20 @@ class TestDocument(FrappeTestCase): self.assertEqual(d.send_reminder, 1) return d + def test_website_route_default(self): + default = frappe.generate_hash() + child_table = new_doctype(default=default, istable=1).insert().name + parent = ( + new_doctype(fields=[{"fieldtype": "Table", "options": child_table, "fieldname": "child_table"}]) + .insert() + .name + ) + + doc = frappe.get_doc({"doctype": parent, "child_table": [{"some_fieldname": "xasd"}]}).insert() + doc.append("child_table", {}) + doc.save() + self.assertEqual(doc.child_table[-1].some_fieldname, default) + def test_insert_with_child(self): d = frappe.get_doc( { From 62d634eab43b96a20324869de47a970a6e05791e Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 22 Dec 2023 11:28:05 +0100 Subject: [PATCH 212/237] Revert "fix: check the correct email account" This reverts commit 2507c6af97c8ae37a166a39c4653b193c78cd6c3. --- frappe/core/doctype/communication/communication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 0321cbe7eb..de2dfb7702 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -438,7 +438,7 @@ class Communication(Document, CommunicationEmailMixin): frappe.db.commit() def parse_email_for_timeline_links(self): - if not frappe.db.get_value("Email Account", self.email_account, "enable_automatic_linking"): + if not frappe.db.get_value("Email Account", filters={"enable_automatic_linking": 1}): return for doctype, docname in parse_email([self.recipients, self.cc, self.bcc]): From f2b9f759abc188df0272b635fdee7672fc08660c Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Fri, 22 Dec 2023 11:41:16 +0100 Subject: [PATCH 213/237] fix: missing comma in translations file --- frappe/translations/de.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index 2fdde556f7..ddec9d352b 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -4749,7 +4749,7 @@ You attached {0},Sie haben {0} angehängt, You removed attachment {0},Sie haben den Anhang {0} entfernt, {0} removed attachment {1},{0} hat den Anhang {1} entfernt, Toggle Full Width,Breite wechseln, -Toggle Theme,Theme wechseln +Toggle Theme,Theme wechseln, Documentation,Dokumentation, About,Über, Search or type a command (Ctrl + G),Suchen oder Befehl eingeben (Strg + G), From fc13ff5f29a23faabf94c646049017cb3f3b1e31 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Sat, 23 Dec 2023 11:39:39 +0530 Subject: [PATCH 214/237] chore: minor fix --- frappe/desk/page/setup_wizard/setup_wizard.js | 16 ++++++----- frappe/public/js/frappe/ui/field_group.js | 27 ++++++++++--------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js index c9f7929b28..75865f6ae4 100644 --- a/frappe/desk/page/setup_wizard/setup_wizard.js +++ b/frappe/desk/page/setup_wizard/setup_wizard.js @@ -349,12 +349,16 @@ frappe.setup.SetupWizardSlide = class SetupWizardSlide extends frappe.ui.Slide { setup_telemetry_events() { let me = this; this.fields.filter(frappe.model.is_value_type).forEach((field) => { - me.get_input(field.fieldname).on("change", function () { - frappe.telemetry.capture(`${field.fieldname}_set`, "setup"); - if (field.fieldname == "enable_telemetry" && !me.get_value("enable_telemetry")) { - frappe.telemetry.disable(); - } - }); + field.fieldname && + me.get_input(field.fieldname)?.on("change", function () { + frappe.telemetry.capture(`${field.fieldname}_set`, "setup"); + if ( + field.fieldname == "enable_telemetry" && + !me.get_value("enable_telemetry") + ) { + frappe.telemetry.disable(); + } + }); }); } }; diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js index 20bb7aadbd..e80dff942a 100644 --- a/frappe/public/js/frappe/ui/field_group.js +++ b/frappe/public/js/frappe/ui/field_group.js @@ -17,7 +17,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { } make() { - var me = this; + let me = this; if (this.fields) { super.make(); this.refresh(); @@ -63,7 +63,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { } catch_enter_as_submit() { - var me = this; + let me = this; $(this.body) .find('input[type="text"], input[type="password"], select') .keypress(function (e) { @@ -77,7 +77,8 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { } get_input(fieldname) { - var field = this.fields_dict[fieldname]; + let field = this.fields_dict[fieldname]; + if (!field) return ""; return $(field.txt ? field.txt : field.input); } @@ -86,14 +87,14 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { } get_values(ignore_errors, check_invalid) { - var ret = {}; - var errors = []; + let ret = {}; + let errors = []; let invalid = []; - for (var key in this.fields_dict) { - var f = this.fields_dict[key]; + for (let key in this.fields_dict) { + let f = this.fields_dict[key]; if (f.get_value) { - var v = f.get_value(); + let v = f.get_value(); if (f.df.reqd && is_null(typeof v === "string" ? strip_html(v) : v)) errors.push(__(f.df.label)); @@ -141,13 +142,13 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { } get_value(key) { - var f = this.fields_dict[key]; + let f = this.fields_dict[key]; return f && (f.get_value ? f.get_value() : null); } set_value(key, val) { return new Promise((resolve) => { - var f = this.fields_dict[key]; + let f = this.fields_dict[key]; if (f) { f.set_value(val).then(() => { f.set_input?.(val); @@ -170,7 +171,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { set_values(dict) { let promises = []; - for (var key in dict) { + for (let key in dict) { if (this.fields_dict[key]) { promises.push(this.set_value(key, dict[key])); } @@ -180,8 +181,8 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { } clear() { - for (var key in this.fields_dict) { - var f = this.fields_dict[key]; + for (let key in this.fields_dict) { + let f = this.fields_dict[key]; if (f && f.set_input) { f.set_input(f.df["default"] || ""); } From 42fbb5862617350917b6875888d7beba07a87d3a Mon Sep 17 00:00:00 2001 From: Corentin Flr <10946971+cogk@users.noreply.github.com> Date: Sat, 23 Dec 2023 18:39:28 +0100 Subject: [PATCH 215/237] fix(document): Pass parent_doc and parentfield in _set_defaults --- frappe/model/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 80593275d2..1201d3755b 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -778,7 +778,7 @@ class Document(BaseDocument): # children for df in self.meta.get_table_fields(): - new_doc = frappe.new_doc(df.options, as_dict=True) + new_doc = frappe.new_doc(df.options, parent_doc=self, parentfield=df.fieldname, as_dict=True) value = self.get(df.fieldname) if isinstance(value, list): for d in value: From ad93afcd4c4565de062d4caa0360c0ad03f8a334 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 25 Dec 2023 18:28:18 +0100 Subject: [PATCH 216/237] fix: use system font for printing This way, content from Text Editor will have the same style as the rest of the print. --- frappe/www/printview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 2095e7aa76..9fb8d89a2d 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -582,7 +582,7 @@ def get_print_style( def get_font( print_settings: "PrintSettings", print_format: Optional["PrintFormat"] = None, for_legacy=False ) -> str: - default = 'Inter, "Helvetica Neue", Helvetica, Arial, "Open Sans", sans-serif' + default = "var(--font-stack)" if for_legacy: return default From 3b0f6de883a9125690473d586260e47a8951b149 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Wed, 25 Oct 2023 13:40:54 +0530 Subject: [PATCH 217/237] perf: don't extract backup files unless required Read from the gzipped file wherever possible Signed-off-by: Akhil Narang --- frappe/commands/site.py | 65 +++++++++------------ frappe/installer.py | 126 ++++++++++++++++++++++++++++------------ frappe/utils/backups.py | 7 ++- 3 files changed, 121 insertions(+), 77 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9b8d2d9131..a6deb0a16c 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -194,8 +194,7 @@ def _restore( _backup = Backup(sql_file_path) try: - decompressed_file_name = extract_sql_from_archive(sql_file_path) - if is_partial(decompressed_file_name): + if is_partial(sql_file_path): click.secho( "Partial Backup file detected. You cannot use a partial file to restore a Frappe site.", fg="red", @@ -218,16 +217,14 @@ def _restore( encryption_key = get_or_generate_backup_encryption_key() _backup.backup_decryption(encryption_key) - # Rollback on unsuccessful decryrption + # Rollback on unsuccessful decryption if not os.path.exists(sql_file_path): click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") _backup.decryption_rollback() sys.exit(1) - decompressed_file_name = extract_sql_from_archive(sql_file_path) - - if is_partial(decompressed_file_name): + if is_partial(sql_file_path): click.secho( "Partial Backup file detected. You cannot use a partial file to restore a Frappe site.", fg="red", @@ -239,16 +236,20 @@ def _restore( _backup.decryption_rollback() sys.exit(1) - validate_database_sql(decompressed_file_name, _raise=not force) - - # dont allow downgrading to older versions of frappe without force - if not force and is_downgrade(decompressed_file_name, verbose=True): + # don't allow downgrading to older versions of frappe without force + if not force and is_downgrade(sql_file_path, verbose=True): warn_message = ( "This is not recommended and may lead to unexpected behaviour. " "Do you want to continue anyway?" ) click.confirm(warn_message, abort=True) + # Extract file if its gzipped + decompressed_file_name = extract_sql_from_archive(sql_file_path) + + # Validate the sql file + validate_database_sql(decompressed_file_name, _raise=not force) + try: _new_site( frappe.conf.db_name, @@ -312,7 +313,7 @@ def _restore( @click.option("--encryption-key", help="Backup encryption key") @pass_context def partial_restore(context, sql_file_path, verbose, encryption_key=None): - from frappe.installer import extract_sql_from_archive, partial_restore + from frappe.installer import is_partial, partial_restore from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key if not os.path.exists(sql_file_path): @@ -328,19 +329,13 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): frappe.connect(site=site) try: - decompressed_file_name = extract_sql_from_archive(sql_file_path) - - with open(decompressed_file_name) as f: - header = " ".join(f.readline() for _ in range(5)) - - # Check for full backup file - if "Partial Backup" not in header: - click.secho( - "Full backup file detected.Use `bench restore` to restore a Frappe Site.", - fg="red", - ) - _backup.decryption_rollback() - sys.exit(1) + if not is_partial(sql_file_path): + click.secho( + "Full backup file detected.Use `bench restore` to restore a Frappe Site.", + fg="red", + ) + _backup.decryption_rollback() + sys.exit(1) except UnicodeDecodeError: _backup.decryption_rollback() @@ -354,25 +349,19 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): _backup.backup_decryption(key) - # Rollback on unsuccessful decryrption + # Rollback on unsuccessful decryption if not os.path.exists(sql_file_path): click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") _backup.decryption_rollback() sys.exit(1) - decompressed_file_name = extract_sql_from_archive(sql_file_path) - - with open(decompressed_file_name) as f: - header = " ".join(f.readline() for _ in range(5)) - - # Check for Full backup file. - if "Partial Backup" not in header: - click.secho( - "Full Backup file detected.Use `bench restore` to restore a Frappe Site.", - fg="red", - ) - _backup.decryption_rollback() - sys.exit(1) + if not is_partial(sql_file_path): + click.secho( + "Full Backup file detected.Use `bench restore` to restore a Frappe Site.", + fg="red", + ) + _backup.decryption_rollback() + sys.exit(1) partial_restore(sql_file_path, verbose) diff --git a/frappe/installer.py b/frappe/installer.py index 891de5f2e8..742a0c454a 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -1,6 +1,7 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - +import configparser +import gzip import json import os import re @@ -52,7 +53,7 @@ def _new_site( ): """Install a new Frappe site""" - from frappe.utils import get_site_path, scheduler, touch_file + from frappe.utils import get_site_path, scheduler if not force and os.path.exists(site): print(f"Site {site} already exists") @@ -793,48 +794,81 @@ def is_downgrade(sql_file_path, verbose=False): from semantic_version import Version - head = "INSERT INTO `tabInstalled Application` VALUES" + backup_version = None + try: + backup_version = extract_version_from_dump(sql_file_path) + except Exception: + # Handle older backups in the same way + head = "INSERT INTO `tabInstalled Application` VALUES" - with open(sql_file_path) as f: - for line in f: - if head in line: - # 'line' (str) format: ('2056588823','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',1,'frappe','v10.1.71-74 (3c50d5e) (v10.x.x)','v10.x.x'),('855c640b8e','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',2,'your_custom_app','0.0.1','master') - line = line.strip().lstrip(head).rstrip(";").strip() - app_rows = frappe.safe_eval(line) - # check if iterable consists of tuples before trying to transform - apps_list = ( - app_rows - if all(isinstance(app_row, (tuple, list, set)) for app_row in app_rows) - else (app_rows,) - ) - # 'all_apps' (list) format: [('frappe', '12.x.x-develop ()', 'develop'), ('your_custom_app', '0.0.1', 'master')] - all_apps = [x[-3:] for x in apps_list] + with open(sql_file_path) as f: + for line in f: + if head in line: + # 'line' (str) format: ('2056588823','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',1,'frappe','v10.1.71-74 (3c50d5e) (v10.x.x)','v10.x.x'),('855c640b8e','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',2,'your_custom_app','0.0.1','master') + line = line.strip().lstrip(head).rstrip(";").strip() + app_rows = frappe.safe_eval(line) + # check if iterable consists of tuples before trying to transform + apps_list = ( + app_rows + if all(isinstance(app_row, (tuple, list, set)) for app_row in app_rows) + else (app_rows,) + ) + # 'all_apps' (list) format: [('frappe', '12.x.x-develop ()', 'develop'), ('your_custom_app', '0.0.1', 'master')] + all_apps = [x[-3:] for x in apps_list] - for app in all_apps: - app_name = app[0] - app_version = app[1].split(" ", 1)[0] + for app in all_apps: + app_name = app[0] + app_version = app[1].split(" ", 1)[0] - if app_name == "frappe": - try: - current_version = Version(frappe.__version__) - backup_version = Version(app_version[1:] if app_version[0] == "v" else app_version) - except ValueError: - return False + if app_name == "frappe": + try: + backup_version = app_version[1:] if app_version[0] == "v" else app_version + break + except ValueError: + return False - downgrade = backup_version > current_version + # Assume it's not a downgrade if we can't determine backup version + if backup_version is None: + return False - if verbose and downgrade: - print(f"Your site will be downgraded from Frappe {backup_version} to {current_version}") + current_version = Version(frappe.__version__) + downgrade = Version(backup_version) > current_version - return downgrade + if verbose and downgrade: + print(f"Your site will be downgraded from Frappe {backup_version} to {current_version}") + + return downgrade -def is_partial(sql_file_path): - with open(sql_file_path) as f: - header = " ".join(f.readline() for _ in range(5)) - if "Partial Backup" in header: - return True - return False +def extract_version_from_dump(sql_file_path: str) -> str | None: + """ + Extract frappe version from DB dump + + :param sql_file_path: The path to the dump file + :return: The frappe version used to create the backup + """ + header = get_db_dump_header(sql_file_path).split("\n") + metadata = "" + if "begin frappe metadata" in header[0]: + for line in header[1:]: + if "end frappe metadata" in line: + break + metadata += line.replace("--", "").strip() + "\n" + parser = configparser.ConfigParser() + parser.read_string(metadata) + return parser["frappe"]["version"] + return None + + +def is_partial(sql_file_path: str) -> bool: + """ + Function to return whether the database dump is a partial backup or not + + :param sql_file_path: path to the database dump file + :return: True if the database dump is a partial backup, False otherwise + """ + header = get_db_dump_header(sql_file_path) + return "Partial Backup" in header def partial_restore(sql_file_path, verbose=False): @@ -877,7 +911,7 @@ def validate_database_sql(path, _raise=True): error_message = f"{path} is an empty file!" empty_file = True - # dont bother checking if empty file + # don't bother checking if empty file if not empty_file: with open(path) as f: for line in f: @@ -893,3 +927,21 @@ def validate_database_sql(path, _raise=True): if _raise and (missing_table or empty_file): raise frappe.InvalidDatabaseFile + + +def get_db_dump_header(file_path: str, file_bytes: int = 256) -> str: + """ + Get the header of a database dump file + + :param file_path: path to the database dump file + :param file_bytes: number of bytes to read from the file + :return: The first few bytes of the file as requested + """ + + # Use `gzip` to open the file if the extension is `.gz` + if file_path.endswith(".gz"): + with gzip.open(file_path, "rb") as f: + return f.read(file_bytes).decode() + + with open(file_path, "rb") as f: + return f.read(file_bytes).decode() diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index e716ff6e7a..48aa060377 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -373,7 +373,11 @@ class BackupGenerator: ) database_header_content = [ - f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}", + "begin frappe metadata", + "[frappe]", + f"version = {frappe.__version__}", + f"branch = {get_app_branch('frappe') or 'N/A'}", + "end frappe metadata", "", ] @@ -673,7 +677,6 @@ def backup( backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, - quiet=False, ): "Backup" odb = scheduled_backup( From cb7c0e653cbdf3e48cfd3686e19076d38b276519 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 26 Oct 2023 15:35:16 +0530 Subject: [PATCH 218/237] fix(Backup): automatically rollback decryption when object is being deleted This allows us to not have to call it everytime before returning Signed-off-by: Akhil Narang --- frappe/utils/backups.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 48aa060377..ba546801e5 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -670,6 +670,9 @@ class Backup: os.remove(self.file_path.rstrip(".gz")) os.rename(self.file_path + ".gpg", self.file_path) + def __del__(self): + self.decryption_rollback() + def backup( with_files=False, From 7db8baa7c40dc5d348c65809d3affcc6f14d110b Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 26 Oct 2023 15:36:32 +0530 Subject: [PATCH 219/237] feat(installer): drop actual gzip file extraction Use `zgrep` to check for table name match where required Also use a table that's at the top of the dump files (`__Auth`) Signed-off-by: Akhil Narang --- frappe/installer.py | 89 ++++++++++++--------------------------------- 1 file changed, 23 insertions(+), 66 deletions(-) diff --git a/frappe/installer.py b/frappe/installer.py index 742a0c454a..a81955833a 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -665,32 +665,6 @@ def remove_missing_apps(): frappe.db.set_global("installed_apps", json.dumps(installed_apps)) -def extract_sql_from_archive(sql_file_path): - """Return the path of an SQL file if the passed argument is the path of a gzipped - SQL file or an SQL file path. The path may be absolute or relative from the bench - root directory or the sites sub-directory. - - Args: - sql_file_path (str): Path of the SQL file - - Return: - str: Path of the decompressed SQL file - """ - from frappe.utils import get_bench_relative_path - - sql_file_path = get_bench_relative_path(sql_file_path) - # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file - if sql_file_path.endswith("sql.gz"): - decompressed_file_name = extract_sql_gzip(sql_file_path) - else: - decompressed_file_name = sql_file_path - - # convert archive sql to latest compatible - convert_archive_content(decompressed_file_name) - - return decompressed_file_name - - def convert_archive_content(sql_file_path): if frappe.conf.db_type == "mariadb": # ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed @@ -724,20 +698,6 @@ def convert_archive_content(sql_file_path): old_sql_file_path.unlink() -def extract_sql_gzip(sql_gz_path): - import subprocess - - try: - original_file = sql_gz_path - decompressed_file = original_file.rstrip(".gz") - cmd = f"gzip --decompress --force < {original_file} > {decompressed_file}" - subprocess.check_call(cmd, shell=True) - except Exception: - raise - - return decompressed_file - - def _guess_mariadb_version() -> tuple[int] | None: # Using command-line because we *might* not have a connection yet and this command is required # in non-interactive mode. @@ -872,8 +832,6 @@ def is_partial(sql_file_path: str) -> bool: def partial_restore(sql_file_path, verbose=False): - sql_file = extract_sql_from_archive(sql_file_path) - if frappe.conf.db_type == "mariadb": from frappe.database.mariadb.setup_db import import_db_from_sql elif frappe.conf.db_type == "postgres": @@ -887,45 +845,44 @@ def partial_restore(sql_file_path, verbose=False): fg="yellow", ) warnings.warn(warn) + else: + click.secho("Unsupported database type", fg="red") + return - import_db_from_sql(source_sql=sql_file, verbose=verbose) - - # Removing temporarily created file - if sql_file != sql_file_path: - os.remove(sql_file) + import_db_from_sql(source_sql=sql_file_path, verbose=verbose) -def validate_database_sql(path, _raise=True): - """Check if file has contents and if DefaultValue table exists +def validate_database_sql(path: str, _raise: bool = True) -> None: + """Check if file has contents and if `__Auth` table exists Args: path (str): Path of the decompressed SQL file _raise (bool, optional): Raise exception if invalid file. Defaults to True. """ - empty_file = False - missing_table = True - error_message = "" + if path.endswith(".gz"): + executable_name = "zgrep" + else: + executable_name = "grep" - if not os.path.getsize(path): + if os.path.getsize(path): + if (executable := which(executable_name)) is None: + frappe.throw( + f"`{executable_name}` not found in PATH! This is required to take a backup.", + exc=frappe.ExecutableNotFound, + ) + try: + frappe.utils.execute_in_shell(f"{executable} -m1 __Auth {path}", check_exit_code=True) + return + except Exception: + error_message = "Table `__Auth` not found in file." + else: error_message = f"{path} is an empty file!" - empty_file = True - - # don't bother checking if empty file - if not empty_file: - with open(path) as f: - for line in f: - if "tabDefaultValue" in line: - missing_table = False - break - - if missing_table: - error_message = "Table `tabDefaultValue` not found in file." if error_message: click.secho(error_message, fg="red") - if _raise and (missing_table or empty_file): + if _raise: raise frappe.InvalidDatabaseFile From 0b508e2a961807ea8784255a64b71cb6afe6ae22 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 26 Oct 2023 15:37:49 +0530 Subject: [PATCH 220/237] feat(db_manager): avoid extraction of DB dump if gzipped Use `gzip -cd` to directly get the contents onto stdout and pipe to mariadb Signed-off-by: Akhil Narang --- frappe/database/db_manager.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 68cd39f2f5..27704fb472 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -61,12 +61,23 @@ class DbManager: command = [] - if pv: - command.extend([pv, source, "|"]) - source = [] - print("Restoring Database file...") + if source.endswith(".gz"): + if gzip := which("gzip"): + source = [] + command.extend([gzip, "-cd", source, "|"]) + if pv: + command.extend([pv, "|"]) + print("Restoring Database file...") + else: + raise Exception("`gzip` not installed") + else: - source = ["<", source] + if pv: + command.extend([pv, source, "|"]) + source = [] + print("Restoring Database file...") + else: + source = ["<", source] bin, args, bin_name = get_command( host=frappe.conf.db_host, From 2b7c74dd5e8ba7d3c7e6ac37a891a1258693d138 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 26 Oct 2023 15:39:33 +0530 Subject: [PATCH 221/237] feat(restore): handle encrypted backups better Determine the mimetype based on the file contents instead of waiting for an exception Cleaner + no need of duplicate code Signed-off-by: Akhil Narang --- frappe/commands/site.py | 102 ++++++++++------------------------------ 1 file changed, 25 insertions(+), 77 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index a6deb0a16c..9a7b659390 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -5,6 +5,7 @@ import sys # imports - third party imports import click +import magic # imports - module imports import frappe @@ -72,13 +73,10 @@ def new_site( setup_db=True, ): "Create a new site" - from frappe.installer import _new_site, extract_sql_from_archive + from frappe.installer import _new_site frappe.init(site=site, new_site=True) - if source_sql: - source_sql = extract_sql_from_archive(source_sql) - _new_site( db_name, site, @@ -180,11 +178,9 @@ def _restore( with_public_files=None, with_private_files=None, ): - from frappe.installer import ( _new_site, extract_files, - extract_sql_from_archive, is_downgrade, is_partial, validate_database_sql, @@ -192,22 +188,8 @@ def _restore( from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key _backup = Backup(sql_file_path) - - try: - if is_partial(sql_file_path): - click.secho( - "Partial Backup file detected. You cannot use a partial file to restore a Frappe site.", - fg="red", - ) - click.secho( - "Use `bench partial-restore` to restore a partial backup to an existing site.", - fg="yellow", - ) - _backup.decryption_rollback() - sys.exit(1) - - except UnicodeDecodeError: - _backup.decryption_rollback() + backup_mimetype = magic.from_file(sql_file_path) + if "cipher" in backup_mimetype: if encryption_key: click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow") _backup.backup_decryption(encryption_key) @@ -220,21 +202,18 @@ def _restore( # Rollback on unsuccessful decryption if not os.path.exists(sql_file_path): click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") - - _backup.decryption_rollback() sys.exit(1) - if is_partial(sql_file_path): - click.secho( - "Partial Backup file detected. You cannot use a partial file to restore a Frappe site.", - fg="red", - ) - click.secho( - "Use `bench partial-restore` to restore a partial backup to an existing site.", - fg="yellow", - ) - _backup.decryption_rollback() - sys.exit(1) + if is_partial(sql_file_path): + click.secho( + "Partial Backup file detected. You cannot use a partial file to restore a Frappe site.", + fg="red", + ) + click.secho( + "Use `bench partial-restore` to restore a partial backup to an existing site.", + fg="yellow", + ) + sys.exit(1) # don't allow downgrading to older versions of frappe without force if not force and is_downgrade(sql_file_path, verbose=True): @@ -244,11 +223,8 @@ def _restore( ) click.confirm(warn_message, abort=True) - # Extract file if its gzipped - decompressed_file_name = extract_sql_from_archive(sql_file_path) - # Validate the sql file - validate_database_sql(decompressed_file_name, _raise=not force) + validate_database_sql(sql_file_path, _raise=not force) try: _new_site( @@ -259,47 +235,35 @@ def _restore( admin_password=admin_password, verbose=verbose, install_apps=install_app, - source_sql=decompressed_file_name, + source_sql=sql_file_path, force=True, db_type=frappe.conf.db_type, ) except Exception as err: print(err.args[1]) - _backup.decryption_rollback() sys.exit(1) - # Removing temporarily created file - if decompressed_file_name != sql_file_path: - os.remove(decompressed_file_name) - _backup.decryption_rollback() - # Extract public and/or private files to the restored site, if user has given the path if with_public_files: # Decrypt data if there is a Key if encryption_key: _backup = Backup(with_public_files) _backup.backup_decryption(encryption_key) - if not os.path.exists(with_public_files): - _backup.decryption_rollback() public = extract_files(site, with_public_files) # Removing temporarily created file os.remove(public) - _backup.decryption_rollback() if with_private_files: # Decrypt data if there is a Key if encryption_key: _backup = Backup(with_private_files) _backup.backup_decryption(encryption_key) - if not os.path.exists(with_private_files): - _backup.decryption_rollback() private = extract_files(site, with_private_files) # Removing temporarily created file os.remove(private) - _backup.decryption_rollback() success_message = "Site {} has been restored{}".format( site, " with files" if (with_public_files or with_private_files) else "" @@ -328,17 +292,9 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): verbose = context.verbose or verbose frappe.connect(site=site) - try: - if not is_partial(sql_file_path): - click.secho( - "Full backup file detected.Use `bench restore` to restore a Frappe Site.", - fg="red", - ) - _backup.decryption_rollback() - sys.exit(1) - - except UnicodeDecodeError: - _backup.decryption_rollback() + _backup = Backup(sql_file_path) + backup_mimetype = magic.from_file(sql_file_path) + if "cipher" in backup_mimetype: if encryption_key: click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow") key = encryption_key @@ -352,24 +308,16 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): # Rollback on unsuccessful decryption if not os.path.exists(sql_file_path): click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") - _backup.decryption_rollback() sys.exit(1) - if not is_partial(sql_file_path): - click.secho( - "Full Backup file detected.Use `bench restore` to restore a Frappe Site.", - fg="red", - ) - _backup.decryption_rollback() - sys.exit(1) + if not is_partial(sql_file_path): + click.secho( + "Full backup file detected.Use `bench restore` to restore a Frappe Site.", + fg="red", + ) + sys.exit(1) partial_restore(sql_file_path, verbose) - - # Removing temporarily created file - _backup.decryption_rollback() - if os.path.exists(sql_file_path.rstrip(".gz")): - os.remove(sql_file_path.rstrip(".gz")) - frappe.destroy() From 50d21677b872ff89d2912c7b57467757a35f549a Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 26 Oct 2023 17:51:34 +0530 Subject: [PATCH 222/237] chore: drop use of `magic` library, run `file` command directly Signed-off-by: Akhil Narang --- frappe/commands/site.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9a7b659390..d35e24a223 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -5,7 +5,6 @@ import sys # imports - third party imports import click -import magic # imports - module imports import frappe @@ -188,8 +187,12 @@ def _restore( from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key _backup = Backup(sql_file_path) - backup_mimetype = magic.from_file(sql_file_path) - if "cipher" in backup_mimetype: + err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True) + if err: + click.secho("Failed to detect type of backup file", fg="red") + sys.exit(1) + + if "cipher" in out.decode().split(":")[-1].strip(): if encryption_key: click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow") _backup.backup_decryption(encryption_key) @@ -293,8 +296,12 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): frappe.connect(site=site) _backup = Backup(sql_file_path) - backup_mimetype = magic.from_file(sql_file_path) - if "cipher" in backup_mimetype: + err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True) + if err: + click.secho("Failed to detect type of backup file", fg="red") + sys.exit(1) + + if "cipher" in out.decode().split(":")[-1].strip(): if encryption_key: click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow") key = encryption_key From 012b0fdb7e5c63a99dfc5f62a7895b450c5afb21 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Wed, 1 Nov 2023 19:11:15 +0530 Subject: [PATCH 223/237] fix(postgres/setup): use `gzip` to get backup contents if the file is an archive Signed-off-by: Akhil Narang --- frappe/database/postgres/setup_db.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 8de3e532b9..35bc51d12a 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -50,12 +50,22 @@ def import_db_from_sql(source_sql=None, verbose=False): command = [] - if pv: - command.extend([pv, source_sql, "|"]) - source = [] - print("Restoring Database file...") + if source_sql.endswith(".gz"): + if gzip := which("gzip"): + source = [] + if pv: + command.extend([gzip, "-cd", source_sql, "|", pv, "|"]) + else: + command.extend([gzip, "-cd", source_sql, "|"]) + else: + raise Exception("`gzip` not installed") else: - source = ["-f", source_sql] + if pv: + command.extend([pv, source_sql, "|"]) + source = [] + print("Restoring Database file...") + else: + source = ["-f", source_sql] bin, args, bin_name = get_command( host=frappe.conf.db_host, From 148efbc3ed1cb5ce087d3c95d1ac1a464559318d Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 2 Nov 2023 20:59:07 +0530 Subject: [PATCH 224/237] refactor(restore): adjust downgrade check First actually check whether its a downgrade, then check for the force flag Signed-off-by: Akhil Narang --- frappe/commands/site.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index d35e24a223..b5253ecf8d 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -218,8 +218,8 @@ def _restore( ) sys.exit(1) - # don't allow downgrading to older versions of frappe without force - if not force and is_downgrade(sql_file_path, verbose=True): + # Check if the backup is of an older version of frappe and the user hasn't specified force + if is_downgrade(sql_file_path, verbose=True) and not force: warn_message = ( "This is not recommended and may lead to unexpected behaviour. " "Do you want to continue anyway?" From 2e382040cd96b7aacbe7926a2227038cffb3649f Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 2 Nov 2023 20:59:39 +0530 Subject: [PATCH 225/237] fix: simplify version detection logic Our version detection code is relatively simple, so we shouldn't have any exceptions arising there Just check for a None return value to decide whether we should use the older logic Signed-off-by: Akhil Narang --- frappe/installer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/installer.py b/frappe/installer.py index a81955833a..461343fd27 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -754,10 +754,8 @@ def is_downgrade(sql_file_path, verbose=False): from semantic_version import Version - backup_version = None - try: - backup_version = extract_version_from_dump(sql_file_path) - except Exception: + backup_version = extract_version_from_dump(sql_file_path) + if backup_version is None: # Handle older backups in the same way head = "INSERT INTO `tabInstalled Application` VALUES" From a06e402f38c4e823ad995749d405bdc488fe3cba Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 6 Nov 2023 13:42:36 +0530 Subject: [PATCH 226/237] refactor: use a function with context manager for backup decryption Signed-off-by: Akhil Narang --- frappe/commands/site.py | 145 ++++++++++++++++++++++++---------------- frappe/utils/backups.py | 62 +++++++---------- 2 files changed, 112 insertions(+), 95 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index b5253ecf8d..859f2cfc85 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -177,16 +177,9 @@ def _restore( with_public_files=None, with_private_files=None, ): - from frappe.installer import ( - _new_site, - extract_files, - is_downgrade, - is_partial, - validate_database_sql, - ) - from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key + from frappe.installer import extract_files + from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key - _backup = Backup(sql_file_path) err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True) if err: click.secho("Failed to detect type of backup file", fg="red") @@ -195,17 +188,79 @@ def _restore( if "cipher" in out.decode().split(":")[-1].strip(): if encryption_key: click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow") - _backup.backup_decryption(encryption_key) else: click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") encryption_key = get_or_generate_backup_encryption_key() - _backup.backup_decryption(encryption_key) - # Rollback on unsuccessful decryption - if not os.path.exists(sql_file_path): - click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") - sys.exit(1) + with decrypt_backup(sql_file_path, encryption_key): + # Rollback on unsuccessful decryption + if not os.path.exists(sql_file_path): + click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") + sys.exit(1) + + restore_backup( + sql_file_path, + site, + db_root_username, + db_root_password, + verbose, + install_app, + admin_password, + force, + ) + else: + restore_backup( + sql_file_path, + site, + db_root_username, + db_root_password, + verbose, + install_app, + admin_password, + force, + ) + + # Extract public and/or private files to the restored site, if user has given the path + if with_public_files: + # Decrypt data if there is a Key + if encryption_key: + with decrypt_backup(with_public_files, encryption_key): + public = extract_files(site, with_public_files) + else: + public = extract_files(site, with_public_files) + + # Removing temporarily created file + os.remove(public) + + if with_private_files: + # Decrypt data if there is a Key + if encryption_key: + with decrypt_backup(with_private_files, encryption_key): + private = extract_files(site, with_private_files) + else: + private = extract_files(site, with_private_files) + + # Removing temporarily created file + os.remove(private) + + success_message = "Site {} has been restored{}".format( + site, " with files" if (with_public_files or with_private_files) else "" + ) + click.secho(success_message, fg="green") + + +def restore_backup( + sql_file_path: str, + site, + db_root_username, + db_root_password, + verbose, + install_app, + admin_password, + force, +): + from frappe.installer import _new_site, is_downgrade, is_partial, validate_database_sql if is_partial(sql_file_path): click.secho( @@ -247,32 +302,6 @@ def _restore( print(err.args[1]) sys.exit(1) - # Extract public and/or private files to the restored site, if user has given the path - if with_public_files: - # Decrypt data if there is a Key - if encryption_key: - _backup = Backup(with_public_files) - _backup.backup_decryption(encryption_key) - public = extract_files(site, with_public_files) - - # Removing temporarily created file - os.remove(public) - - if with_private_files: - # Decrypt data if there is a Key - if encryption_key: - _backup = Backup(with_private_files) - _backup.backup_decryption(encryption_key) - private = extract_files(site, with_private_files) - - # Removing temporarily created file - os.remove(private) - - success_message = "Site {} has been restored{}".format( - site, " with files" if (with_public_files or with_private_files) else "" - ) - click.secho(success_message, fg="green") - @click.command("partial-restore") @click.argument("sql-file-path") @@ -281,21 +310,16 @@ def _restore( @pass_context def partial_restore(context, sql_file_path, verbose, encryption_key=None): from frappe.installer import is_partial, partial_restore - from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key + from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key if not os.path.exists(sql_file_path): print("Invalid path", sql_file_path) sys.exit(1) site = get_site(context) - frappe.init(site=site) - - _backup = Backup(sql_file_path) - verbose = context.verbose or verbose - + frappe.init(site=site) frappe.connect(site=site) - _backup = Backup(sql_file_path) err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True) if err: click.secho("Failed to detect type of backup file", fg="red") @@ -310,21 +334,30 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") key = get_or_generate_backup_encryption_key() - _backup.backup_decryption(key) + with decrypt_backup(sql_file_path, key): + if not is_partial(sql_file_path): + click.secho( + "Full backup file detected.Use `bench restore` to restore a Frappe Site.", + fg="red", + ) + sys.exit(1) + + partial_restore(sql_file_path, verbose) # Rollback on unsuccessful decryption if not os.path.exists(sql_file_path): click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") sys.exit(1) - if not is_partial(sql_file_path): - click.secho( - "Full backup file detected.Use `bench restore` to restore a Frappe Site.", - fg="red", - ) - sys.exit(1) + else: + if not is_partial(sql_file_path): + click.secho( + "Full backup file detected.Use `bench restore` to restore a Frappe Site.", + fg="red", + ) + sys.exit(1) - partial_restore(sql_file_path, verbose) + partial_restore(sql_file_path, verbose) frappe.destroy() diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index ba546801e5..0e52580927 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -1,5 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import contextlib # imports - standard imports import gzip @@ -632,46 +633,29 @@ def get_or_generate_backup_encryption_key(): return key -class Backup: - def __init__(self, file_path): - self.file_path = file_path +@contextlib.contextmanager +def decrypt_backup(file_path: str, passphrase: str): + if not os.path.exists(file_path): + print("Invalid path: ", file_path) + return + else: + file_path_with_ext = file_path + ".gpg" + os.rename(file_path, file_path_with_ext) - def backup_decryption(self, passphrase): - """ - Decrypts backup at the given path using the passphrase. - """ - if not os.path.exists(self.file_path): - print("Invalid path", self.file_path) - return - else: - file_path_with_ext = self.file_path + ".gpg" - os.rename(self.file_path, file_path_with_ext) - - cmd_string = "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -o {decrypted_file} -d {file_location}" - command = cmd_string.format( - passphrase=passphrase, - file_location=file_path_with_ext, - decrypted_file=self.file_path, - ) - frappe.utils.execute_in_shell(command) - - def decryption_rollback(self): - """ - Checks if the decrypted file exists at the given path. - if exists - Renames the orginal encrypted file. - else - Removes the decrypted file and rename the original file. - """ - if os.path.exists(self.file_path + ".gpg"): - if os.path.exists(self.file_path): - os.remove(self.file_path) - if os.path.exists(self.file_path.rstrip(".gz")): - os.remove(self.file_path.rstrip(".gz")) - os.rename(self.file_path + ".gpg", self.file_path) - - def __del__(self): - self.decryption_rollback() + cmd_string = "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -o {decrypted_file} -d {file_location}" + command = cmd_string.format( + passphrase=passphrase, + file_location=file_path_with_ext, + decrypted_file=file_path, + ) + frappe.utils.execute_in_shell(command) + yield + if os.path.exists(file_path + ".gpg"): + if os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(file_path.rstrip(".gz")): + os.remove(file_path.rstrip(".gz")) + os.rename(file_path + ".gpg", file_path) def backup( From 998f2c10d67f5d6af35d2f35872bd5442e40d71b Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 6 Nov 2023 13:57:46 +0530 Subject: [PATCH 227/237] fix: handle older gzipped backups as well Also fix the actual comparison Signed-off-by: Akhil Narang --- frappe/installer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frappe/installer.py b/frappe/installer.py index 461343fd27..22913fe2ee 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -53,7 +53,7 @@ def _new_site( ): """Install a new Frappe site""" - from frappe.utils import get_site_path, scheduler + from frappe.utils import scheduler if not force and os.path.exists(site): print(f"Site {site} already exists") @@ -756,11 +756,17 @@ def is_downgrade(sql_file_path, verbose=False): backup_version = extract_version_from_dump(sql_file_path) if backup_version is None: + if sql_file_path.endswith(".sql.gz"): + open_method = gzip.open + else: + open_method = open # Handle older backups in the same way head = "INSERT INTO `tabInstalled Application` VALUES" - with open(sql_file_path) as f: + with open_method(sql_file_path) as f: for line in f: + if isinstance(line, bytes): + line = line.decode("utf-8").strip() if head in line: # 'line' (str) format: ('2056588823','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',1,'frappe','v10.1.71-74 (3c50d5e) (v10.x.x)','v10.x.x'),('855c640b8e','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',2,'your_custom_app','0.0.1','master') line = line.strip().lstrip(head).rstrip(";").strip() @@ -790,7 +796,7 @@ def is_downgrade(sql_file_path, verbose=False): return False current_version = Version(frappe.__version__) - downgrade = Version(backup_version) > current_version + downgrade = Version(backup_version) < current_version if verbose and downgrade: print(f"Your site will be downgraded from Frappe {backup_version} to {current_version}") From 7f433b84afef89d716f20a702075eff25eb76dda Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 9 Nov 2023 18:13:23 +0530 Subject: [PATCH 228/237] feat: allow creating a backup with the older metadata style Signed-off-by: Akhil Narang --- frappe/commands/site.py | 5 +++++ frappe/utils/backups.py | 28 ++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 859f2cfc85..48a4feea57 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -842,6 +842,9 @@ def use(site, sites_path="."): ) @click.option("--verbose", default=False, is_flag=True, help="Add verbosity") @click.option("--compress", default=False, is_flag=True, help="Compress private and public files") +@click.option( + "--old-backup-metadata", default=False, is_flag=True, help="Use older backup metadata" +) @pass_context def backup( context, @@ -856,6 +859,7 @@ def backup( compress=False, include="", exclude="", + old_backup_metadata=False, ): "Backup" @@ -881,6 +885,7 @@ def backup( compress=compress, verbose=verbose, force=True, + old_backup_metadata=old_backup_metadata, ) except Exception: click.secho( diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 0e52580927..bdba23ac2f 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -55,6 +55,7 @@ class BackupGenerator: include_doctypes="", exclude_doctypes="", verbose=False, + old_backup_metadata=False, ): global _verbose self.compress_files = compress_files or compress @@ -73,6 +74,7 @@ class BackupGenerator: self.include_doctypes = include_doctypes self.exclude_doctypes = exclude_doctypes self.partial = False + self.old_backup_metadata = old_backup_metadata site = frappe.local.site or frappe.generate_hash(length=8) self.site_slug = site.replace(".", "_") @@ -373,14 +375,20 @@ class BackupGenerator: _("gzip not found in PATH! This is required to take a backup."), exc=frappe.ExecutableNotFound ) - database_header_content = [ - "begin frappe metadata", - "[frappe]", - f"version = {frappe.__version__}", - f"branch = {get_app_branch('frappe') or 'N/A'}", - "end frappe metadata", - "", - ] + if self.old_backup_metadata: + database_header_content = [ + f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}", + "", + ] + else: + database_header_content = [ + "begin frappe metadata", + "[frappe]", + f"version = {frappe.__version__}", + f"branch = {get_app_branch('frappe') or 'N/A'}", + "end frappe metadata", + "", + ] if self.backup_includes: backup_info = ("Backing Up Tables: ", ", ".join(self.backup_includes)) @@ -516,6 +524,7 @@ def scheduled_backup( compress=False, force=False, verbose=False, + old_backup_metadata=False, ): """this function is called from scheduler deletes backups older than 7 days @@ -534,6 +543,7 @@ def scheduled_backup( compress=compress, force=force, verbose=verbose, + old_backup_metadata=old_backup_metadata, ) @@ -551,6 +561,7 @@ def new_backup( compress=False, force=False, verbose=False, + old_backup_metadata=False, ): delete_temp_backups() odb = BackupGenerator( @@ -570,6 +581,7 @@ def new_backup( exclude_doctypes=exclude_doctypes, verbose=verbose, compress_files=compress, + old_backup_metadata=old_backup_metadata, ) odb.get_backup(older_than, ignore_files, force=force) return odb From 9bf818eb4a655adf945431f46fa9a7ca8d13f3c7 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 9 Nov 2023 18:21:56 +0530 Subject: [PATCH 229/237] chore: add in some tests for backup Signed-off-by: Akhil Narang --- frappe/tests/test_commands.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index e6af22dd0c..1de2365713 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -547,6 +547,42 @@ class TestBackups(BaseTestCommands): self.assertIn("successfully completed", self.stdout) self.assertNotEqual(before_backup["database"], after_backup["database"]) + @skipIf( + not (frappe.conf.db_type == "mariadb"), + "Only for MariaDB", + ) + def test_backup_extract_restore(self): + """Restore a backup after extracting""" + self.execute("bench --site {site} backup") + self.assertEqual(self.returncode, 0) + backup = fetch_latest_backups() + self.execute(f"gunzip {backup['database']}") + self.assertEqual(self.returncode, 0) + backup_sql = backup["database"].replace(".gz", "") + assert os.path.isfile(backup_sql) + self.execute( + "bench --site {site} restore {backup_sql}", + { + "backup_sql": backup_sql, + }, + ) + self.assertEqual(self.returncode, 0) + + @skipIf( + not (frappe.conf.db_type == "mariadb"), + "Only for MariaDB", + ) + def test_old_backup_restore(self): + """Restore a backup after extracting""" + self.execute("bench --site {site} backup --old-backup-metadata") + self.assertEqual(self.returncode, 0) + backup = fetch_latest_backups() + self.execute( + "bench --site {site} restore {database}", + backup, + ) + self.assertEqual(self.returncode, 0) + def test_backup_fails_with_exit_code(self): """Provide incorrect options to check if exit code is 1""" odb = BackupGenerator( From b010dc584f4c86941f3405025c4b0991a610ca70 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 20 Nov 2023 16:40:51 +0530 Subject: [PATCH 230/237] chore(installer): fix output when prompting user about downgrade Signed-off-by: Akhil Narang --- frappe/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/installer.py b/frappe/installer.py index 22913fe2ee..b8962a9cb9 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -799,7 +799,7 @@ def is_downgrade(sql_file_path, verbose=False): downgrade = Version(backup_version) < current_version if verbose and downgrade: - print(f"Your site will be downgraded from Frappe {backup_version} to {current_version}") + print(f"Your site will be downgraded from Frappe {current_version} to {backup_version}") return downgrade From 76c9fbd0cddbd507704906d4f6667b010984bc2b Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 20 Nov 2023 16:50:30 +0530 Subject: [PATCH 231/237] fix: use source before changing its contents Signed-off-by: Akhil Narang --- frappe/database/db_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 27704fb472..05aff3f9ab 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -63,8 +63,8 @@ class DbManager: if source.endswith(".gz"): if gzip := which("gzip"): - source = [] command.extend([gzip, "-cd", source, "|"]) + source = [] if pv: command.extend([pv, "|"]) print("Restoring Database file...") From 3fe840fb31de57727eaa22ef200ad9f1e90f2602 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Tue, 21 Nov 2023 11:00:53 +0530 Subject: [PATCH 232/237] fix(postgres): make use of common helper Signed-off-by: Akhil Narang --- frappe/database/postgres/setup_db.py | 55 ++++------------------------ 1 file changed, 8 insertions(+), 47 deletions(-) diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 35bc51d12a..f5f3b14006 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -1,7 +1,7 @@ import os import frappe -from frappe import _ +from frappe.database.db_manager import DbManager def setup_database(): @@ -36,55 +36,16 @@ def bootstrap_database(db_name, verbose, source_sql=None): def import_db_from_sql(source_sql=None, verbose=False): - import shlex - from shutil import which - - from frappe.database import get_command - from frappe.utils import execute_in_shell - - # bootstrap db + if verbose: + print("Starting database import...") + db_name = frappe.conf.db_name if not source_sql: source_sql = os.path.join(os.path.dirname(__file__), "framework_postgres.sql") - - pv = which("pv") - - command = [] - - if source_sql.endswith(".gz"): - if gzip := which("gzip"): - source = [] - if pv: - command.extend([gzip, "-cd", source_sql, "|", pv, "|"]) - else: - command.extend([gzip, "-cd", source_sql, "|"]) - else: - raise Exception("`gzip` not installed") - else: - if pv: - command.extend([pv, source_sql, "|"]) - source = [] - print("Restoring Database file...") - else: - source = ["-f", source_sql] - - bin, args, bin_name = get_command( - host=frappe.conf.db_host, - port=frappe.conf.db_port, - user=frappe.conf.db_name, - password=frappe.conf.db_password, - db_name=frappe.conf.db_name, + DbManager(frappe.local.db).restore_database( + verbose, db_name, source_sql, db_name, frappe.conf.db_password ) - - if not bin: - frappe.throw( - _("{} not found in PATH! This is required to restore the database.").format(bin_name), - exc=frappe.ExecutableNotFound, - ) - command.append(bin) - command.append(shlex.join(args)) - command.extend(source) - execute_in_shell(" ".join(command), check_exit_code=True, verbose=verbose) - frappe.cache.delete_keys("") # Delete all keys associated with this site. + if verbose: + print("Imported from database %s" % source_sql) def get_root_connection(root_login=None, root_password=None): From 6e8f32af58d3e0dd5d1c15709fa59ec7730647b1 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Thu, 30 Nov 2023 13:54:05 +0530 Subject: [PATCH 233/237] chore: don't pipe output through `pv` No point if we're using `execute_in_shell()` Signed-off-by: Akhil Narang --- frappe/database/db_manager.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 05aff3f9ab..01c18d69c4 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -57,27 +57,17 @@ class DbManager: from frappe.database import get_command from frappe.utils import execute_in_shell - pv = which("pv") - command = [] if source.endswith(".gz"): if gzip := which("gzip"): command.extend([gzip, "-cd", source, "|"]) source = [] - if pv: - command.extend([pv, "|"]) - print("Restoring Database file...") else: raise Exception("`gzip` not installed") else: - if pv: - command.extend([pv, source, "|"]) - source = [] - print("Restoring Database file...") - else: - source = ["<", source] + source = ["<", source] bin, args, bin_name = get_command( host=frappe.conf.db_host, From 43021911ffc6d7c014e17c89874fd9e3b214fe73 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 4 Dec 2023 12:28:29 +0530 Subject: [PATCH 234/237] fix: simplify parsing version for older backups Signed-off-by: Akhil Narang --- frappe/installer.py | 39 ++++----------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/frappe/installer.py b/frappe/installer.py index b8962a9cb9..d96f1167f1 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -450,7 +450,6 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]: def _delete_linked_documents( module_name: str, doctype_linkfield_map: dict[str, str], dry_run: bool ) -> None: - """Deleted all records linked with module def""" for doctype, fieldname in doctype_linkfield_map.items(): for record in frappe.get_all(doctype, filters={fieldname: module_name}, pluck="name"): @@ -756,40 +755,10 @@ def is_downgrade(sql_file_path, verbose=False): backup_version = extract_version_from_dump(sql_file_path) if backup_version is None: - if sql_file_path.endswith(".sql.gz"): - open_method = gzip.open - else: - open_method = open - # Handle older backups in the same way - head = "INSERT INTO `tabInstalled Application` VALUES" - - with open_method(sql_file_path) as f: - for line in f: - if isinstance(line, bytes): - line = line.decode("utf-8").strip() - if head in line: - # 'line' (str) format: ('2056588823','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',1,'frappe','v10.1.71-74 (3c50d5e) (v10.x.x)','v10.x.x'),('855c640b8e','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',2,'your_custom_app','0.0.1','master') - line = line.strip().lstrip(head).rstrip(";").strip() - app_rows = frappe.safe_eval(line) - # check if iterable consists of tuples before trying to transform - apps_list = ( - app_rows - if all(isinstance(app_row, (tuple, list, set)) for app_row in app_rows) - else (app_rows,) - ) - # 'all_apps' (list) format: [('frappe', '12.x.x-develop ()', 'develop'), ('your_custom_app', '0.0.1', 'master')] - all_apps = [x[-3:] for x in apps_list] - - for app in all_apps: - app_name = app[0] - app_version = app[1].split(" ", 1)[0] - - if app_name == "frappe": - try: - backup_version = app_version[1:] if app_version[0] == "v" else app_version - break - except ValueError: - return False + # This is likely an older backup, so try to extract another way + header = get_db_dump_header(sql_file_path).split("\n") + if "Version" in header[0]: + backup_version = header[0].split(":")[-1].strip() # Assume it's not a downgrade if we can't determine backup version if backup_version is None: From 68d4a5ad82a28bcfcd20ca7545e5d1cb6d8bc469 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Tue, 26 Dec 2023 14:42:19 +0800 Subject: [PATCH 235/237] Do not call begin in updatedb after syncing (#23934) Do not call begin in updatedb after syncing as commit does it already. Signed-off-by: Xiaoguang Sun --- frappe/database/mariadb/database.py | 2 +- frappe/database/postgres/database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 1096821e66..00cbd1c332 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -440,7 +440,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): self.commit() db_table.sync() - self.begin() + self.commit() def get_database_list(self): return self.sql("SHOW DATABASES", pluck=True) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 617350cca5..48dd55381a 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -329,7 +329,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): self.commit() db_table.sync() - self.begin() + self.commit() @staticmethod def get_on_duplicate_update(key="name"): From 854cebd5b1ffec3ad1463e2e42fdd01e5fbfddf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8C=9B=E7=81=AB?= Date: Tue, 26 Dec 2023 17:04:48 +0800 Subject: [PATCH 236/237] fix: Solve the problem that the document editor will report an error when the type name includes multiple spaces (#23940) --- frappe/public/js/form_builder/components/Field.vue | 2 +- frappe/public/js/form_builder/components/FieldProperties.vue | 2 +- frappe/public/js/workflow_builder/components/Properties.vue | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/form_builder/components/Field.vue b/frappe/public/js/form_builder/components/Field.vue index 26d488cdf8..9514bbea8e 100644 --- a/frappe/public/js/form_builder/components/Field.vue +++ b/frappe/public/js/form_builder/components/Field.vue @@ -30,7 +30,7 @@ const label_input = ref(null); const hovered = ref(false); const selected = computed(() => store.selected(props.field.df.name)); const component = computed(() => { - return props.field.df.fieldtype.replace(" ", "") + "Control"; + return props.field.df.fieldtype.replaceAll(" ", "") + "Control"; }); function remove_field() { diff --git a/frappe/public/js/form_builder/components/FieldProperties.vue b/frappe/public/js/form_builder/components/FieldProperties.vue index 5f903ed36c..159d8eb43f 100644 --- a/frappe/public/js/form_builder/components/FieldProperties.vue +++ b/frappe/public/js/form_builder/components/FieldProperties.vue @@ -86,7 +86,7 @@ let docfield_df = computed(() => {
{
Date: Tue, 26 Dec 2023 19:50:33 +0800 Subject: [PATCH 237/237] fix: Solve the problem that there is no forEach instance method in HTMLCollection in some environments, which leads to the failure of camera-related functions. --- frappe/public/js/frappe/ui/capture.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/capture.js b/frappe/public/js/frappe/ui/capture.js index 63095c2b7f..1d40f50485 100644 --- a/frappe/public/js/frappe/ui/capture.js +++ b/frappe/public/js/frappe/ui/capture.js @@ -234,7 +234,7 @@ frappe.ui.Capture = class { setup_remove_action() { let me = this; - let elements = this.$template[0].getElementsByClassName("capture-remove-btn"); + let elements = Array.from(this.$template[0].getElementsByClassName("capture-remove-btn")); elements.forEach((el) => { el.onclick = () => {