From e86df2ecf6ee84b78a0745e61f88e73dfef774f1 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 29 May 2023 17:19:11 +0530 Subject: [PATCH 01/29] feat: Scroll to field via URL --- frappe/public/js/frappe/form/form.js | 10 +++++++--- frappe/public/js/frappe/utils/utils.js | 25 ++++++++++++++----------- frappe/public/scss/desk/global.scss | 16 +++++++++++++++- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 47917422b5..e33c4ab7d3 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1414,8 +1414,12 @@ frappe.ui.form.Form = class FrappeForm { if (selector.length) { frappe.utils.scroll_to(selector); } - } else if (window.location.hash && $(window.location.hash).length) { - frappe.utils.scroll_to(window.location.hash, true, 200, null, null, true); + } else if (window.location.hash) { + if ($(window.location.hash).length) { + frappe.utils.scroll_to(window.location.hash, true, 200, null, null, true); + } else { + this.scroll_to_field(window.location.hash.replace("#", "")); + } } } @@ -1926,7 +1930,7 @@ frappe.ui.form.Form = class FrappeForm { } // highlight control inside field - let control_element = $el.find(".form-control"); + let control_element = $el.closest(".frappe-control"); control_element.addClass("highlight"); setTimeout(() => { control_element.removeClass("highlight"); diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index ac9a18785b..7020513f47 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -340,9 +340,21 @@ Object.assign(frappe.utils, { scroll_top = 0; } + const highlight = () => { + if (highlight_element) { + $(element).addClass("highlight"); + document.addEventListener( + "click", + function () { + $(element).removeClass("highlight"); + }, + { once: true } + ); + } + }; // already there if (scroll_top == element_to_be_scrolled.scrollTop()) { - return; + return highlight(); } if (animate) { @@ -352,16 +364,7 @@ Object.assign(frappe.utils, { }) .promise() .then(() => { - if (highlight_element) { - $(element).addClass("highlight"); - document.addEventListener( - "click", - function () { - $(element).removeClass("highlight"); - }, - { once: true } - ); - } + highlight(); callback && callback(); }); } else { diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index 765e51cab9..08715c28b0 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -571,11 +571,25 @@ details > summary:focus { display: none; } -.highlight { +.highlight:not(.frappe-control) { transition: 0.5s ease background-color; box-shadow: var(--highlight-shadow) !important; } +.frappe-control.highlight { + --wrap-padding: calc(-1 * var(--padding-sm)); + &::after { + content: " "; + border-radius: 5px; + box-shadow: var(--highlight-shadow) !important; + top: var(--wrap-padding); + position: absolute; + bottom: var(--wrap-padding); + left: var(--wrap-padding); + right: var(--wrap-padding); + } +} + .dropdown-menu.small { font-size: var(--text-sm); min-width: 140px; From b9d4974360fe6b35823c54ff97cec6cb58c5d2a4 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 29 May 2023 17:34:38 +0530 Subject: [PATCH 02/29] fix: Clear hash once the scrolling is done --- frappe/public/js/frappe/form/form.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index e33c4ab7d3..322776ada9 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1418,7 +1418,8 @@ frappe.ui.form.Form = class FrappeForm { if ($(window.location.hash).length) { frappe.utils.scroll_to(window.location.hash, true, 200, null, null, true); } else { - this.scroll_to_field(window.location.hash.replace("#", "")); + this.scroll_to_field(window.location.hash.replace("#", "")) && + history.replaceState(null, null, " "); } } } @@ -1935,6 +1936,7 @@ frappe.ui.form.Form = class FrappeForm { setTimeout(() => { control_element.removeClass("highlight"); }, 2000); + return true; } setup_docinfo_change_listener() { From 70944cabb33ba770b19c4b923026c50166bc544f Mon Sep 17 00:00:00 2001 From: Vimal Patel Date: Wed, 7 Jun 2023 12:25:52 +0100 Subject: [PATCH 03/29] fix: New Dashboard Chart throws TypeError: format requires a mapping closes #21250 --- frappe/desk/query_report.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 6f4bc716aa..814f556a0d 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -165,7 +165,8 @@ def get_script(report_name): script += f"\n\n//# sourceURL={scrub(report.name)}__custom" if not script: - script = "frappe.query_reports['%s']={}" % report_name + filters = json.dumps([prepare_filter(filter) for filter in report.filters]) + script = "frappe.query_reports['%s']={ 'filters': %s}" % (report_name, filters) return { "script": render_include(script), @@ -174,6 +175,18 @@ def get_script(report_name): } +def prepare_filter(filter): + filter = { + "fieldname": filter.fieldname, + "label": filter.label, + "fieldtype": filter.fieldtype, + "width": "80", + "options": filter.options, + "reqd": 1 + } + return filter + + @frappe.whitelist() @frappe.read_only() def run( From 0d729e5e47c4a27204877af8b11e466cac72d823 Mon Sep 17 00:00:00 2001 From: Vimal Patel Date: Thu, 8 Jun 2023 11:07:22 +0100 Subject: [PATCH 04/29] fix: New Dashboard Chart throws TypeError: format requires a mapping closes #21250 backport version-14-hotfix --- frappe/desk/query_report.py | 6 +++--- frappe/public/js/frappe/views/reports/report_utils.js | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 814f556a0d..fb8e39a495 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -165,13 +165,13 @@ def get_script(report_name): script += f"\n\n//# sourceURL={scrub(report.name)}__custom" if not script: - filters = json.dumps([prepare_filter(filter) for filter in report.filters]) - script = "frappe.query_reports['%s']={ 'filters': %s}" % (report_name, filters) + script = "frappe.query_reports['%s']={}" % report_name return { "script": render_include(script), "html_format": html_format, - "execution_time": frappe.cache.hget("report_execution_time", report_name) or 0, + "execution_time": frappe.cache().hget("report_execution_time", report_name) or 0, + "filters": report.filters } diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js index 9713f8bb99..c1402aa562 100644 --- a/frappe/public/js/frappe/views/reports/report_utils.js +++ b/frappe/public/js/frappe/views/reports/report_utils.js @@ -126,6 +126,13 @@ frappe.report_utils = { .then((r) => { frappe.dom.eval(r.script || ""); return frappe.after_ajax(() => { + if ( + frappe.query_reports[report_name] && + !frappe.query_reports[report_name].filter && + r.filters + ) { + return frappe.query_reports[report_name].filters = r.filters + } return ( frappe.query_reports[report_name] && frappe.query_reports[report_name].filters From 48b9128904d84cdaefbf333e78ccdcccc931b104 Mon Sep 17 00:00:00 2001 From: Vimal Patel Date: Thu, 8 Jun 2023 11:26:22 +0100 Subject: [PATCH 05/29] fix: remove redundant prepare_filter method --- frappe/desk/query_report.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index fb8e39a495..9e003c0e2f 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -170,23 +170,11 @@ def get_script(report_name): return { "script": render_include(script), "html_format": html_format, - "execution_time": frappe.cache().hget("report_execution_time", report_name) or 0, + "execution_time": frappe.cache.hget("report_execution_time", report_name) or 0, "filters": report.filters } -def prepare_filter(filter): - filter = { - "fieldname": filter.fieldname, - "label": filter.label, - "fieldtype": filter.fieldtype, - "width": "80", - "options": filter.options, - "reqd": 1 - } - return filter - - @frappe.whitelist() @frappe.read_only() def run( From 428eaf6a1681c32c629815e41a9e80a5b0d14f9f Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Thu, 8 Jun 2023 16:52:24 +0530 Subject: [PATCH 06/29] chore: linter fix --- frappe/desk/query_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 9e003c0e2f..5b7c450ae9 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -171,7 +171,7 @@ def get_script(report_name): "script": render_include(script), "html_format": html_format, "execution_time": frappe.cache.hget("report_execution_time", report_name) or 0, - "filters": report.filters + "filters": report.filters, } From 108e128b46bbf3183833503f21c57774fcc732be Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Thu, 8 Jun 2023 16:52:52 +0530 Subject: [PATCH 07/29] chore: linter fix --- frappe/public/js/frappe/views/reports/report_utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js index c1402aa562..1d33701147 100644 --- a/frappe/public/js/frappe/views/reports/report_utils.js +++ b/frappe/public/js/frappe/views/reports/report_utils.js @@ -131,7 +131,7 @@ frappe.report_utils = { !frappe.query_reports[report_name].filter && r.filters ) { - return frappe.query_reports[report_name].filters = r.filters + return (frappe.query_reports[report_name].filters = r.filters) } return ( frappe.query_reports[report_name] && From fa502278f42c0551f8d18f84431b827560a4b730 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Thu, 8 Jun 2023 16:53:17 +0530 Subject: [PATCH 08/29] chore --- frappe/public/js/frappe/views/reports/report_utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js index 1d33701147..d09cbfd285 100644 --- a/frappe/public/js/frappe/views/reports/report_utils.js +++ b/frappe/public/js/frappe/views/reports/report_utils.js @@ -131,7 +131,7 @@ frappe.report_utils = { !frappe.query_reports[report_name].filter && r.filters ) { - return (frappe.query_reports[report_name].filters = r.filters) + return (frappe.query_reports[report_name].filters = r.filters); } return ( frappe.query_reports[report_name] && From 666813158c2618fff0ce65ef014d2af09d8de562 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Thu, 8 Jun 2023 17:12:03 +0530 Subject: [PATCH 09/29] chore: linter fix --- frappe/public/js/frappe/views/reports/report_utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js index d09cbfd285..d75716541b 100644 --- a/frappe/public/js/frappe/views/reports/report_utils.js +++ b/frappe/public/js/frappe/views/reports/report_utils.js @@ -132,7 +132,7 @@ frappe.report_utils = { r.filters ) { return (frappe.query_reports[report_name].filters = r.filters); - } + } return ( frappe.query_reports[report_name] && frappe.query_reports[report_name].filters From 155465c58def8a67fbf678c512d43a2cef6f8a7a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 9 Jun 2023 00:35:20 +0530 Subject: [PATCH 10/29] fix: currency precision formatter (#21293) --- cypress/integration/control_currency.js | 74 +++++++++++++++++++ .../js/frappe/form/controls/currency.js | 2 +- frappe/public/js/frappe/form/formatters.js | 12 ++- 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 cypress/integration/control_currency.js diff --git a/cypress/integration/control_currency.js b/cypress/integration/control_currency.js new file mode 100644 index 0000000000..5e6db86036 --- /dev/null +++ b/cypress/integration/control_currency.js @@ -0,0 +1,74 @@ +context("Control Currency", () => { + const fieldname = "currency_field"; + + before(() => { + cy.login(); + cy.visit("/app/website"); + }); + + function get_dialog_with_currency(df_options = {}) { + return cy.dialog({ + title: "Currency Check", + fields: [ + { + fieldname: fieldname, + fieldtype: "Currency", + Label: "Currency", + ...df_options, + }, + ], + }); + } + + it("check value changes", () => { + const TEST_CASES = [ + { + input: "10.101", + df_options: { precision: 1 }, + blur_expected: "10.1", + }, + { + input: "10.101", + df_options: { precision: "3" }, + blur_expected: "10.101", + }, + { + input: "10.101", + df_options: { precision: "" }, // default assumed to be 2; + blur_expected: "10.10", + }, + { + input: "10.101", + df_options: { precision: "0" }, + blur_expected: "10", + }, + { + input: "10.101", + df_options: { precision: 0 }, + blur_expected: "10", + }, + { + input: "10.101", + df_options: { precision: "" }, + blur_expected: "10.1", + default_precision: 1, + }, + ]; + + TEST_CASES.forEach((test_case) => { + cy.window() + .its("frappe") + .then((frappe) => { + frappe.boot.sysdefaults.currency = test_case.currency; + frappe.boot.sysdefaults.currency_precision = test_case.default_precision ?? 2; + }); + + get_dialog_with_currency(test_case.df_options).as("dialog"); + cy.get_field(fieldname, "Currency").clear(); + cy.wait(300); + cy.fill_field(fieldname, test_case.input, "Currency").blur(); + cy.get_field(fieldname, "Currency").should("have.value", test_case.blur_expected); + cy.hide_dialog(); + }); + }); +}); diff --git a/frappe/public/js/frappe/form/controls/currency.js b/frappe/public/js/frappe/form/controls/currency.js index a7d30c071b..e2f79ba446 100644 --- a/frappe/public/js/frappe/form/controls/currency.js +++ b/frappe/public/js/frappe/form/controls/currency.js @@ -7,7 +7,7 @@ frappe.ui.form.ControlCurrency = class ControlCurrency extends frappe.ui.form.Co get_precision() { // always round based on field precision or currency's precision // this method is also called in this.parse() - if (!this.df.precision) { + if (typeof this.df.precision != "number" && !this.df.precision) { if (frappe.boot.sysdefaults.currency_precision) { this.df.precision = frappe.boot.sysdefaults.currency_precision; } else { diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 637fd7063d..beddbf512d 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -103,9 +103,15 @@ frappe.form.formatters = { }, Currency: function (value, docfield, options, doc) { var currency = frappe.meta.get_field_currency(docfield, doc); - var precision = cint( - docfield.precision ?? frappe.boot.sysdefaults.currency_precision ?? 2 - ); + + let precision; + if (typeof docfield.precision == "number") { + precision = docfield.precision; + } else { + precision = cint( + docfield.precision || frappe.boot.sysdefaults.currency_precision || 2 + ); + } // If you change anything below, it's going to hurt a company in UAE, a bit. if (precision > 2) { From 6188d1476e13027bb5edbb838619dee6168c995c Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 9 Jun 2023 10:02:43 +0530 Subject: [PATCH 11/29] fix: correct method for layout reset --- frappe/custom/doctype/customize_form/customize_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 3937079365..1a7bb9070a 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -179,7 +179,7 @@ frappe.ui.form.on("Customize Form", { () => { return frm.call({ doc: frm.doc, - method: "reset_to_defaults", + method: "reset_layout", callback: function (r) { if (!r.exc) { frappe.show_alert({ From 15ce89fb72fbcbe517b7141fa1e911f3bcc98d3c Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 9 Jun 2023 10:40:05 +0530 Subject: [PATCH 12/29] fix: minor improvements to `CustomizeForm.reset_layout` (#21296) --- frappe/custom/doctype/customize_form/customize_form.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index f403079cd8..edc30727a5 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -578,12 +578,13 @@ class CustomizeForm(Document): filters={ "doc_type": self.doc_type, "property": "field_order", - "is_system_generated": False, }, ) - if property_setter: - frappe.delete_doc("Property Setter", property_setter) + if not property_setter: + return + + frappe.delete_doc("Property Setter", property_setter) self.fetch_to_customize() @classmethod From 345da6e3147aeafcf022e238e45e3c21cf319eb7 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 9 Jun 2023 12:19:30 +0530 Subject: [PATCH 13/29] feat: support reordering standard fields in Form Builder (#21297) --- frappe/public/js/form_builder/FormBuilder.vue | 5 ++--- frappe/public/js/form_builder/components/Column.vue | 4 +--- frappe/public/js/form_builder/components/Section.vue | 5 +---- frappe/public/js/form_builder/components/Tabs.vue | 10 ++-------- frappe/public/js/form_builder/store.js | 5 +++++ 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/frappe/public/js/form_builder/FormBuilder.vue b/frappe/public/js/form_builder/FormBuilder.vue index 2a1441c51a..c1ab90c4a4 100644 --- a/frappe/public/js/form_builder/FormBuilder.vue +++ b/frappe/public/js/form_builder/FormBuilder.vue @@ -166,8 +166,7 @@ onMounted(() => { } } - :deep([data-has-std-field="false"]), - :deep([data-is-custom="1"]) { + :deep([data-is-user-generated="1"]) { background-color: var(--yellow-highlight-color); } } @@ -175,7 +174,7 @@ onMounted(() => { :deep(.preview) { --field-placeholder-color: var(--fg-bg-color); - .tab, .column, .field, [data-is-custom="1"] { + .tab, .column, .field { background-color: var(--fg-color); } diff --git a/frappe/public/js/form_builder/components/Column.vue b/frappe/public/js/form_builder/components/Column.vue index acb1ff735e..1563d1033e 100644 --- a/frappe/public/js/form_builder/components/Column.vue +++ b/frappe/public/js/form_builder/components/Column.vue @@ -148,8 +148,6 @@ function move_columns_to_section() { :style="{ backgroundColor: column.fields.length ? '' : 'var(--field-placeholder-color)' }" v-model="column.fields" group="fields" - filter="[data-is-custom='0']" - :prevent-on-filter="false" :animation="200" :easing="store.get_animation" item-key="id" @@ -159,7 +157,7 @@ function move_columns_to_section() { diff --git a/frappe/public/js/form_builder/components/Section.vue b/frappe/public/js/form_builder/components/Section.vue index c97fe1e4d8..5131ff25d3 100644 --- a/frappe/public/js/form_builder/components/Section.vue +++ b/frappe/public/js/form_builder/components/Section.vue @@ -160,8 +160,6 @@ function move_sections_to_tab() { backgroundColor: section.columns.length ? null : 'var(--field-placeholder-color)' }" v-model="section.columns" - filter="[data-has-std-field='true']" - :prevent-on-filter="false" group="columns" item-key="id" :animation="200" @@ -172,8 +170,7 @@ function move_sections_to_tab() { diff --git a/frappe/public/js/form_builder/components/Tabs.vue b/frappe/public/js/form_builder/components/Tabs.vue index b587d9d37e..5c233dbd1b 100644 --- a/frappe/public/js/form_builder/components/Tabs.vue +++ b/frappe/public/js/form_builder/components/Tabs.vue @@ -114,8 +114,6 @@ function delete_tab(with_children) { class="tabs" v-model="store.form.layout.tabs" group="tabs" - filter="[data-has-std-field='true']" - :prevent-on-filter="false" :animation="200" :easing="store.get_animation" item-key="id" @@ -125,8 +123,7 @@ function delete_tab(with_children) {
diff --git a/frappe/public/js/form_builder/store.js b/frappe/public/js/form_builder/store.js index b60bdc7919..384baf646d 100644 --- a/frappe/public/js/form_builder/store.js +++ b/frappe/public/js/form_builder/store.js @@ -64,6 +64,10 @@ export const useStore = defineStore("form-builder-store", () => { }); } + function is_user_generated_field(field) { + return cint(field.df.is_custom_field && !field.df.is_system_generated); + } + async function fetch() { await frappe.model.clear_doc("DocType", doctype.value); await frappe.model.with_doctype(doctype.value); @@ -320,6 +324,7 @@ export const useStore = defineStore("form-builder-store", () => { selected, get_df, has_standard_field, + is_user_generated_field, fetch, reset_changes, validate_fields, From 774a75dbdeea16cd14eb215bbe4e7b1c0213b465 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 9 Jun 2023 12:43:40 +0530 Subject: [PATCH 14/29] perf: use cached workflow doc for transitions (#21300) * fix: expire `RECORDER_INTERCEPT_FLAG` in one hour Leaving this running in prod can be dangerous. * perf: Use cached doc in workflows --- frappe/model/workflow.py | 2 +- frappe/recorder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index 1a52077331..0e345a6515 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -210,7 +210,7 @@ def validate_workflow(doc): def get_workflow(doctype) -> "Workflow": - return frappe.get_doc("Workflow", get_workflow_name(doctype)) + return frappe.get_cached_doc("Workflow", get_workflow_name(doctype)) def has_approval_access(user, doc, transition): diff --git a/frappe/recorder.py b/frappe/recorder.py index 8229b862af..1f31181d53 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -169,7 +169,7 @@ def status(*args, **kwargs): @do_not_record @administrator_only def start(*args, **kwargs): - frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1) + frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1, expires_in_sec=60 * 60) @frappe.whitelist() From ed3b764db9c6f432c46f3c76dace5aed6d7f8460 Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Sat, 10 Jun 2023 07:53:15 +0200 Subject: [PATCH 15/29] build(deps): bump cryptography and pyOpenSSL (#21307) cryptography to 41.0.1, pyOpenSSL to 23.2.0 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aa89eed928..62ac3623d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "cairocffi==1.2.0", "chardet~=5.1.0", "croniter~=1.3.5", - "cryptography~=39.0.1", + "cryptography~=41.0.1", "email-reply-parser~=0.5.12", "git-url-parse~=1.2.2", "gunicorn~=20.1.0", @@ -51,7 +51,7 @@ dependencies = [ "premailer~=3.8.0", "psutil~=5.9.1", "psycopg2-binary~=2.9.1", - "pyOpenSSL~=23.0.0", + "pyOpenSSL~=23.2.0", "pycryptodome~=3.10.1", "pydantic~=1.10.2", "pyotp~=2.6.0", From 895dab85352ae9de686aa833d98829df130196b1 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 10 Jun 2023 17:33:42 +0530 Subject: [PATCH 16/29] fix: Pass URL hash separately --- frappe/public/js/frappe/router.js | 35 ++++++++++++++++++------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index c03bfc9b95..9ad917a57a 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -31,20 +31,21 @@ window.addEventListener("popstate", (e) => { return false; }); -// routing v2, capture all clicks so that the target is managed with push-state +// Capture all clicks so that the target is managed with push-state $("body").on("click", "a", function (e) { - let override = (route) => { + const target_element = e.currentTarget; + const href = target_element.getAttribute("href"); + const is_on_same_host = target_element.hostname === window.location.hostname; + + const override = (route) => { e.preventDefault(); frappe.set_route(route); return false; }; - const target_element = e.currentTarget; - const href = target_element.getAttribute("href"); - const is_on_same_host = target_element.hostname === window.location.hostname; - // click handled, but not by href if ( + !is_on_same_host || // external link target_element.getAttribute("onclick") || // has a handler e.ctrlKey || e.metaKey || // open in a new tab @@ -53,18 +54,13 @@ $("body").on("click", "a", function (e) { return; } - if (href === "") { - return override("/app"); - } - if (href && href.startsWith("#")) { // target startswith "#", this is a v1 style route, so remake it. return override(target_element.hash); } - if (is_on_same_host && frappe.router.is_app_route(target_element.pathname)) { + if (frappe.router.is_app_route(target_element.pathname)) { // target has "/app, this is a v2 style route. - if (target_element.search) { frappe.route_options = {}; let params = new URLSearchParams(target_element.search); @@ -72,7 +68,10 @@ $("body").on("click", "a", function (e) { frappe.route_options[key] = value; } } - return override(target_element.pathname + target_element.hash); + if (target_element.hash) { + frappe.route_hash = target_element.hash; + } + return override(target_element.pathname); } }); @@ -139,6 +138,12 @@ frappe.router = { if (!frappe.app) return; let sub_path = this.get_sub_path(); + if (frappe.boot.setup_complete) { + !frappe.re_route["setup-wizard"] && (frappe.re_route["setup-wizard"] = "app"); + } else if (!sub_path.startsWith("setup-wizard")) { + frappe.re_route["setup-wizard"] && delete frappe.re_route["setup-wizard"]; + frappe.set_route(["setup-wizard"]); + } if (this.re_route(sub_path)) return; this.current_sub_path = sub_path; @@ -346,8 +351,8 @@ frappe.router = { route = this.get_route_from_arguments(route); route = this.convert_from_standard_route(route); let sub_path = this.make_url(route); - // replace each # occurrences in the URL with encoded character except for last - // sub_path = sub_path.replace(/[#](?=.*[#])/g, "%23"); + sub_path += frappe.route_hash || ""; + frappe.route_hash = null; if (frappe.open_in_new_tab) { localStorage["route_options"] = JSON.stringify(frappe.route_options); window.open(sub_path, "_blank"); From 67cd951013ddcc259b1175824c45ac1f0c953e98 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 10 Jun 2023 19:22:57 +0530 Subject: [PATCH 17/29] build(deps)!: Require NodeJS 18 as minimum version (#21303) --- .github/workflows/linters.yml | 2 +- .github/workflows/on_release.yml | 2 +- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/server-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- frappe/utils/boilerplate.py | 2 +- package.json | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index c563f9e43f..481041ed68 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -25,7 +25,7 @@ jobs: fetch-depth: 200 - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Check commit titles diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 851b5b1d6a..c17a7c6639 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 4b487d2aea..291c80fc3e 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -69,7 +69,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Add to Hosts diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 4feaebe15d..f42c3bc55c 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -16,7 +16,7 @@ jobs: path: 'frappe' - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - uses: actions/setup-python@v4 with: python-version: '3.11' diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 3b76da1973..c965417928 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -90,7 +90,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Add to Hosts diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 1b88bc73ce..bea00748e9 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Add to Hosts diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 0d786972fb..281d015731 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -582,7 +582,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Cache pip diff --git a/package.json b/package.json index 6cf9d1d60b..d84fb30fdf 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "url": "https://github.com/frappe/frappe/issues" }, "engines": { - "node": ">=14" + "node": ">=18" }, "homepage": "https://frappeframework.com", "dependencies": { From 84294900e54457be0b6655ee11f1f92abd4c359e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 10 Jun 2023 18:20:52 +0530 Subject: [PATCH 18/29] build!: Remove `setup.py` --- setup.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 7a90eed81a..0000000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -# TODO: Remove this file when bench >=v5.11.0 is adopted / v15.0.0 is released -from setuptools import setup - -name = "frappe" - -setup() From 8485ac5d625c70266581796cb4582956188092c5 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 3 Jun 2023 23:19:02 +0530 Subject: [PATCH 19/29] build(deps): bump many dependencies test: change code to adapt to new werkzeug client fix: avoid setting charset utf8 is default and assumed now by werkzeug, setting this manually is deprecated. fix: use string instead of bytes for setting headers DeprecationWarning: Passing bytes as a header value is deprecated and will not be supported in Werkzeug 3.0. 12:23:34 web.1 | response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") --- frappe/tests/test_oauth20.py | 8 ++++---- frappe/utils/response.py | 3 --- frappe/website/utils.py | 8 +++----- pyproject.toml | 24 ++++++++++++------------ 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/frappe/tests/test_oauth20.py b/frappe/tests/test_oauth20.py index 8de652b888..52e9f1b0c5 100644 --- a/frappe/tests/test_oauth20.py +++ b/frappe/tests/test_oauth20.py @@ -107,7 +107,7 @@ class TestOAuth20(FrappeRequestTestCase): update_client_for_auth_code_grant(self.client_id) # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { @@ -154,7 +154,7 @@ class TestOAuth20(FrappeRequestTestCase): update_client_for_auth_code_grant(self.client_id) # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { @@ -203,7 +203,7 @@ class TestOAuth20(FrappeRequestTestCase): frappe.db.commit() # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { @@ -321,7 +321,7 @@ class TestOAuth20(FrappeRequestTestCase): nonce = frappe.generate_hash() # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { diff --git a/frappe/utils/response.py b/frappe/utils/response.py index 79a6b16d1a..b0f8fbc6e6 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -68,7 +68,6 @@ def build_response(response_type=None): def as_csv(): response = Response() response.mimetype = "text/csv" - response.charset = "utf-8" response.headers["Content-Disposition"] = ( 'attachment; filename="%s.csv"' % frappe.response["doctype"].replace(" ", "_") ).encode("utf-8") @@ -79,7 +78,6 @@ def as_csv(): def as_txt(): response = Response() response.mimetype = "text" - response.charset = "utf-8" response.headers["Content-Disposition"] = ( 'attachment; filename="%s.txt"' % frappe.response["doctype"].replace(" ", "_") ).encode("utf-8") @@ -109,7 +107,6 @@ def as_json(): del frappe.local.response["http_status_code"] response.mimetype = "application/json" - response.charset = "utf-8" response.data = json.dumps(frappe.local.response, default=json_handler, separators=(",", ":")) return response diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 922abbb751..de639d1709 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -12,7 +12,7 @@ from werkzeug.wrappers import Response import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint, get_assets_json, get_system_timezone, md_to_html +from frappe.utils import cint, cstr, get_assets_json, get_system_timezone, md_to_html FRONTMATTER_PATTERN = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M) H1_TAG_PATTERN = re.compile("

([^<]*)") @@ -529,14 +529,14 @@ def build_response(path, data, http_status_code, headers: dict | None = None): response = Response() response.data = set_content_type(response, data, path) response.status_code = http_status_code - response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") + response.headers["X-Page-Name"] = cstr(path.encode("ascii", errors="xmlcharrefreplace")) response.headers["X-From-Cache"] = frappe.local.response.from_cache or False add_preload_for_bundled_assets(response) if headers: for key, val in headers.items(): - response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") + response.headers[key] = cstr(val.encode("ascii", errors="xmlcharrefreplace")) return response @@ -544,12 +544,10 @@ def build_response(path, data, http_status_code, headers: dict | None = None): def set_content_type(response, data, path): if isinstance(data, dict): response.mimetype = "application/json" - response.charset = "utf-8" data = json.dumps(data) return data response.mimetype = "text/html" - response.charset = "utf-8" # ignore paths ending with .com to avoid unnecessary download # https://bugs.python.org/issue22347 diff --git a/pyproject.toml b/pyproject.toml index 62ac3623d2..356916eae0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,8 @@ dependencies = [ "filetype~=1.2.0", "GitPython~=3.1.30", "Jinja2~=3.1.2", - "Pillow~=9.3.0", - "PyJWT~=2.4.0", + "Pillow~=9.5.0", + "PyJWT~=2.7.0", "PyMySQL==1.0.3", "PyPDF2~=2.1.0", "PyPika~=0.48.9", @@ -24,14 +24,14 @@ dependencies = [ "PyYAML~=6.0", "RestrictedPython~=6.0", "WeasyPrint==52.5", - "Werkzeug~=2.2.2", + "Werkzeug~=2.3.4", "Whoosh~=2.7.4", "beautifulsoup4~=4.9.3", "bleach-allowlist~=1.0.3", "bleach~=3.3.0", "cairocffi==1.2.0", "chardet~=5.1.0", - "croniter~=1.3.5", + "croniter~=1.3.15", "cryptography~=41.0.1", "email-reply-parser~=0.5.12", "git-url-parse~=1.2.2", @@ -49,20 +49,20 @@ dependencies = [ "pdfkit~=1.0.0", "phonenumbers==8.12.40", "premailer~=3.8.0", - "psutil~=5.9.1", + "psutil~=5.9.5", "psycopg2-binary~=2.9.1", "pyOpenSSL~=23.2.0", "pycryptodome~=3.10.1", - "pydantic~=1.10.2", + "pydantic~=1.10.8", "pyotp~=2.6.0", "python-dateutil~=2.8.1", - "pytz==2022.1", + "pytz==2023.3", "rauth~=0.7.3", - "redis~=4.5.4", - "hiredis~=2.0.0", + "redis~=4.5.5", + "hiredis~=2.2.3", "requests-oauthlib~=1.3.0", "requests~=2.31.0", - "rq~=1.11.1", + "rq~=1.15.0", "rsa>=4.1", "semantic-version~=2.10.0", "sqlparse~=0.4.1", @@ -71,7 +71,7 @@ dependencies = [ "traceback-with-variables~=2.0.4", "xlrd~=2.0.1", "zxcvbn~=4.4.28", - "markdownify~=0.11.2", + "markdownify~=0.11.6", # integration dependencies "boto3~=1.18.49", @@ -103,5 +103,5 @@ coverage = "~=6.5.0" Faker = "~=13.12.1" pyngrok = "~=5.0.5" unittest-xml-reporting = "~=3.0.4" -watchdog = "~=2.1.9" +watchdog = "~=3.0.0" hypothesis = "~=6.68.2" From 699e7446174de4d293bea52a12c9f5ba2d780e92 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 10 Jun 2023 12:26:16 +0530 Subject: [PATCH 20/29] ci: restart bench in patch test also show bench output --- .github/helper/install.sh | 2 +- .github/workflows/patch-mariadb-tests.yml | 6 ++++++ .github/workflows/server-tests.yml | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 39880e35e7..5cdcbebe1a 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -54,7 +54,7 @@ fi echo "Starting Bench..." -bench start &> bench_start.log & +bench start &> ~/frappe-bench/bench_start.log & if [ "$TYPE" == "server" ] then diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 291c80fc3e..eac55106f2 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -126,8 +126,10 @@ jobs: git checkout -q -f $branch_name pip install -U frappe-bench + pgrep honcho | xargs kill rm -rf ~/frappe-bench/env bench -v setup env + bench start &> ~/frappe-bench/bench_start.log & bench --site test_site migrate } @@ -143,3 +145,7 @@ jobs: rm -rf ~/frappe-bench/env bench -v setup env bench --site test_site migrate + + - name: Show bench output + if: ${{ always() }} + run: cat ~/frappe-bench/bench_start.log || true diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index c965417928..f5eac8e380 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -136,6 +136,10 @@ jobs: BUILD_NUMBER: ${{ matrix.container }} TOTAL_BUILDS: 2 + - name: Show bench output + if: ${{ always() }} + run: cat ~/frappe-bench/bench_start.log || true + - name: Upload coverage data uses: actions/upload-artifact@v3 with: From 18e791a353a6af469fca16fc41b909b6e6bf0a81 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 10 Jun 2023 17:56:38 +0530 Subject: [PATCH 21/29] build(deps): PyPDF2 -> pypdf closes https://github.com/frappe/frappe/issues/19861 --- frappe/tests/test_pdf.py | 4 ++-- frappe/utils/pdf.py | 2 +- frappe/utils/print_format.py | 2 +- pyproject.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/tests/test_pdf.py b/frappe/tests/test_pdf.py index 4a8fef253c..84004dc1f1 100644 --- a/frappe/tests/test_pdf.py +++ b/frappe/tests/test_pdf.py @@ -2,7 +2,7 @@ # License: MIT. See LICENSE import io -from PyPDF2 import PdfReader +from pypdf import PdfReader import frappe import frappe.utils.pdf as pdfgen @@ -43,7 +43,7 @@ class TestPdf(FrappeTestCase): password = "qwe" pdf = pdfgen.get_pdf(self.html, options={"password": password}) reader = PdfReader(io.BytesIO(pdf)) - self.assertTrue(reader.isEncrypted) + self.assertTrue(reader.is_encrypted) self.assertTrue(reader.decrypt(password)) def test_pdf_generation_as_a_user(self): diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index 9b7c9a6ce4..721b061257 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -8,7 +8,7 @@ from distutils.version import LooseVersion import pdfkit from bs4 import BeautifulSoup -from PyPDF2 import PdfReader, PdfWriter +from pypdf import PdfReader, PdfWriter import frappe from frappe import _ diff --git a/frappe/utils/print_format.py b/frappe/utils/print_format.py index b96809f2c2..4faaf97780 100644 --- a/frappe/utils/print_format.py +++ b/frappe/utils/print_format.py @@ -1,6 +1,6 @@ import os -from PyPDF2 import PdfWriter +from pypdf import PdfWriter import frappe from frappe import _ diff --git a/pyproject.toml b/pyproject.toml index 356916eae0..5339e194d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "Pillow~=9.5.0", "PyJWT~=2.7.0", "PyMySQL==1.0.3", - "PyPDF2~=2.1.0", + "pypdf~=3.9.1", "PyPika~=0.48.9", "PyQRCode~=1.2.1", "PyYAML~=6.0", From 828490e01a3d14e1b0ac3385ea196c72ab2cc950 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 10 Jun 2023 17:56:26 +0530 Subject: [PATCH 22/29] chore: bump many more old packages --- .../doctype/doctype/boilerplate/__init__.py | 0 pyproject.toml | 40 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) create mode 100644 frappe/core/doctype/doctype/boilerplate/__init__.py diff --git a/frappe/core/doctype/doctype/boilerplate/__init__.py b/frappe/core/doctype/doctype/boilerplate/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pyproject.toml b/pyproject.toml index 5339e194d7..687a08ac0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "Click~=8.1.3", "filelock~=3.8.0", "filetype~=1.2.0", - "GitPython~=3.1.30", + "GitPython~=3.1.31", "Jinja2~=3.1.2", "Pillow~=9.5.0", "PyJWT~=2.7.0", @@ -23,13 +23,13 @@ dependencies = [ "PyQRCode~=1.2.1", "PyYAML~=6.0", "RestrictedPython~=6.0", - "WeasyPrint==52.5", + "WeasyPrint==59.0", "Werkzeug~=2.3.4", "Whoosh~=2.7.4", - "beautifulsoup4~=4.9.3", + "beautifulsoup4~=4.12.2", "bleach-allowlist~=1.0.3", "bleach~=3.3.0", - "cairocffi==1.2.0", + "cairocffi==1.5.1", "chardet~=5.1.0", "croniter~=1.3.15", "cryptography~=41.0.1", @@ -39,35 +39,35 @@ dependencies = [ "html5lib~=1.1", "ipython~=8.10.0", "ldap3~=2.9", - "markdown2~=2.4.0", + "markdown2~=2.4.8", "MarkupSafe>=2.1.0,<3", "maxminddb-geolite2==2018.703", - "num2words~=0.5.10", - "oauthlib~=3.2.1", - "openpyxl~=3.0.7", + "num2words~=0.5.12", + "oauthlib~=3.2.2", + "openpyxl~=3.1.2", "passlib~=1.7.4", "pdfkit~=1.0.0", - "phonenumbers==8.12.40", + "phonenumbers==8.13.13", "premailer~=3.8.0", "psutil~=5.9.5", "psycopg2-binary~=2.9.1", "pyOpenSSL~=23.2.0", - "pycryptodome~=3.10.1", + "pycryptodome~=3.18.0", "pydantic~=1.10.8", - "pyotp~=2.6.0", - "python-dateutil~=2.8.1", + "pyotp~=2.8.0", + "python-dateutil~=2.8.2", "pytz==2023.3", "rauth~=0.7.3", "redis~=4.5.5", "hiredis~=2.2.3", - "requests-oauthlib~=1.3.0", + "requests-oauthlib~=1.3.1", "requests~=2.31.0", "rq~=1.15.0", "rsa>=4.1", "semantic-version~=2.10.0", - "sqlparse~=0.4.1", - "tenacity~=8.0.1", - "terminaltables~=3.1.0", + "sqlparse~=0.4.4", + "tenacity~=8.2.2", + "terminaltables~=3.1.10", "traceback-with-variables~=2.0.4", "xlrd~=2.0.1", "zxcvbn~=4.4.28", @@ -100,8 +100,8 @@ indent = "\t" [tool.bench.dev-dependencies] coverage = "~=6.5.0" -Faker = "~=13.12.1" -pyngrok = "~=5.0.5" -unittest-xml-reporting = "~=3.0.4" +Faker = "~=18.10.1" +pyngrok = "~=6.0.0" +unittest-xml-reporting = "~=3.2.0" watchdog = "~=3.0.0" -hypothesis = "~=6.68.2" +hypothesis = "~=6.77.0" From 3414c0d063827813811bcb293a3333e79ed4aba3 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 10 Jun 2023 19:27:43 +0530 Subject: [PATCH 23/29] refactor!: Merge redis_socketio and redis_queue - realtime communication uses pub-sub and no storage. So using same redis server for both should be just fine. - This is how FC works since quite a lot of time. We haven't seen any problem so far. --- esbuild/esbuild.js | 8 ++++---- frappe/realtime.py | 5 ++--- frappe/utils/connections.py | 2 +- node_utils.js | 2 +- socketio.js | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 4804f0e25f..8c386c86e4 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -446,9 +446,9 @@ function run_build_command_for_apps(apps) { async function notify_redis({ error, success, changed_files }) { // notify redis which in turns tells socketio to publish this to browser - let subscriber = get_redis_subscriber("redis_socketio"); + let subscriber = get_redis_subscriber("redis_queue"); subscriber.on("error", (_) => { - log_warn("Cannot connect to redis_socketio for browser events"); + log_warn("Cannot connect to redis_queue for browser events"); }); let payload = null; @@ -482,9 +482,9 @@ async function notify_redis({ error, success, changed_files }) { } function open_in_editor() { - let subscriber = get_redis_subscriber("redis_socketio"); + let subscriber = get_redis_subscriber("redis_queue"); subscriber.on("error", (_) => { - log_warn("Cannot connect to redis_socketio for open_in_editor events"); + log_warn("Cannot connect to redis_queue for open_in_editor events"); }); subscriber.on("message", (event, file) => { if (event === "open_in_editor") { diff --git a/frappe/realtime.py b/frappe/realtime.py index fdb86546f3..d64b520f56 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE -import os from contextlib import suppress import redis @@ -109,12 +108,12 @@ def emit_via_redis(event, message, room): def get_redis_server(): - """returns redis_socketio connection.""" + """returns redis connection for sending realtime events.""" global redis_server if not redis_server: from redis import Redis - redis_server = Redis.from_url(frappe.conf.redis_socketio or "redis://localhost:12311") + redis_server = Redis.from_url(frappe.conf.redis_queue or "redis://localhost:12311") return redis_server diff --git a/frappe/utils/connections.py b/frappe/utils/connections.py index 6660e6ce19..020bc8b97f 100644 --- a/frappe/utils/connections.py +++ b/frappe/utils/connections.py @@ -3,7 +3,7 @@ from urllib.parse import urlparse from frappe import get_conf -REDIS_KEYS = ("redis_cache", "redis_queue", "redis_socketio") +REDIS_KEYS = ("redis_cache", "redis_queue") def is_open(ip, port, timeout=10): diff --git a/node_utils.js b/node_utils.js index 10744387ca..0b8c455875 100644 --- a/node_utils.js +++ b/node_utils.js @@ -38,7 +38,7 @@ function get_conf() { return conf; } -function get_redis_subscriber(kind = "redis_socketio", options = {}) { +function get_redis_subscriber(kind = "redis_queue", options = {}) { const conf = get_conf(); const host = conf[kind] || conf.redis_async_broker_port; return redis.createClient({ url: host, ...options }); diff --git a/socketio.js b/socketio.js index 8e87a0cce1..31be39c847 100644 --- a/socketio.js +++ b/socketio.js @@ -181,7 +181,7 @@ io.on("connection", function (socket) { }); socket.on("open_in_editor", (data) => { - let s = get_redis_subscriber("redis_socketio"); + let s = get_redis_subscriber("redis_queue"); s.publish("open_in_editor", JSON.stringify(data)); }); }); From eb17c12dda5ee7e920a1f1db20a896491b883f54 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 11 Jun 2023 11:02:22 +0530 Subject: [PATCH 24/29] fix: reuse redis connection if not rq auth --- frappe/utils/background_jobs.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 6a203f8dc7..ea3ea4c191 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -28,6 +28,9 @@ RQ_JOB_FAILURE_TTL = 7 * 24 * 60 * 60 # 7 days instead of 1 year (default) RQ_RESULTS_TTL = 10 * 60 +_redis_queue_conn = None + + @lru_cache def get_queues_timeout(): common_site_config = frappe.get_conf() @@ -47,9 +50,6 @@ def get_queues_timeout(): } -redis_connection = None - - def enqueue( method: str | Callable, queue: str = "default", @@ -360,7 +360,7 @@ def get_redis_conn(username=None, password=None): elif not frappe.local.conf.redis_queue: raise Exception("redis_queue missing in common_site_config.json") - global redis_connection + global _redis_queue_conn cred = frappe._dict() if frappe.conf.get("use_rq_auth"): @@ -374,8 +374,14 @@ def get_redis_conn(username=None, password=None): elif os.environ.get("RQ_ADMIN_PASWORD"): cred["username"] = "default" cred["password"] = os.environ.get("RQ_ADMIN_PASWORD") + try: - redis_connection = RedisQueue.get_connection(**cred) + if not cred: + if not _redis_queue_conn: + _redis_queue_conn = RedisQueue.get_connection() + return _redis_queue_conn + else: + return RedisQueue.get_connection(**cred) except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError): log( f'Wrong credentials used for {cred.username or "default user"}. ' @@ -387,8 +393,6 @@ def get_redis_conn(username=None, password=None): log(f"Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}", colour="red") raise - return redis_connection - def get_queues() -> list[Queue]: """Get all the queues linked to the current bench.""" From 687b660370a24756a9ffc720d370a81e9b3a2644 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 11 Jun 2023 11:04:32 +0530 Subject: [PATCH 25/29] perf: Share queue connection for realtime --- frappe/realtime.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/frappe/realtime.py b/frappe/realtime.py index d64b520f56..376ce4d18b 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -8,8 +8,6 @@ import redis import frappe from frappe.utils.data import cstr -redis_server = None - def publish_progress(percent, title=None, doctype=None, docname=None, description=None): publish_realtime( @@ -101,22 +99,13 @@ def emit_via_redis(event, message, room): :param event: Event name, like `task_progress` etc. :param message: JSON message object. For async must contain `task_id` :param room: name of the room""" + from frappe.utils.background_jobs import get_redis_conn with suppress(redis.exceptions.ConnectionError): - r = get_redis_server() + r = get_redis_conn() r.publish("events", frappe.as_json({"event": event, "message": message, "room": room})) -def get_redis_server(): - """returns redis connection for sending realtime events.""" - global redis_server - if not redis_server: - from redis import Redis - - redis_server = Redis.from_url(frappe.conf.redis_queue or "redis://localhost:12311") - return redis_server - - @frappe.whitelist(allow_guest=True) def can_subscribe_doc(doctype: str, docname: str) -> bool: from frappe.exceptions import PermissionError From 4f797f0bcd70bee30b6f18e120ab84772f39dbba Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 11 Jun 2023 11:11:36 +0530 Subject: [PATCH 26/29] fix: realtime log when commit/rollback happen in same req --- frappe/realtime.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/realtime.py b/frappe/realtime.py index 376ce4d18b..410112b164 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -86,11 +86,12 @@ def flush_realtime_log(): for args in frappe.local._realtime_log: frappe.realtime.emit_via_redis(*args) - frappe.local._realtime_log = [] + clear_realtime_log() def clear_realtime_log(): - frappe.local._realtime_log = [] + if hasattr(frappe.local, "_realtime_log"): + del frappe.local._realtime_log def emit_via_redis(event, message, room): From 6f4a7ddf10fcde15b766d62defaaee6c7a18b3d5 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 11 Jun 2023 11:38:55 +0530 Subject: [PATCH 27/29] Revert "build(deps)!: Require NodeJS 18 as minimum version (#21303)" This reverts commit 67cd951013ddcc259b1175824c45ac1f0c953e98. --- .github/workflows/linters.yml | 2 +- .github/workflows/on_release.yml | 2 +- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/server-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- frappe/utils/boilerplate.py | 2 +- package.json | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 481041ed68..c563f9e43f 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -25,7 +25,7 @@ jobs: fetch-depth: 200 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 16 check-latest: true - name: Check commit titles diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index c17a7c6639..851b5b1d6a 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 16 - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index eac55106f2..c8fe91d287 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -69,7 +69,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 16 check-latest: true - name: Add to Hosts diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index f42c3bc55c..4feaebe15d 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -16,7 +16,7 @@ jobs: path: 'frappe' - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 16 - uses: actions/setup-python@v4 with: python-version: '3.11' diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index f5eac8e380..8ae0be0197 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -90,7 +90,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 16 check-latest: true - name: Add to Hosts diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index bea00748e9..1b88bc73ce 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 16 check-latest: true - name: Add to Hosts diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 281d015731..0d786972fb 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -582,7 +582,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 16 check-latest: true - name: Cache pip diff --git a/package.json b/package.json index d84fb30fdf..6cf9d1d60b 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "url": "https://github.com/frappe/frappe/issues" }, "engines": { - "node": ">=18" + "node": ">=14" }, "homepage": "https://frappeframework.com", "dependencies": { From 32f54b6734c028650c36170fba160698bb2eed77 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 12 Jun 2023 12:13:34 +0530 Subject: [PATCH 28/29] chore: remove socketio from boilerplate workflow [skip ci] --- frappe/utils/boilerplate.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 0d786972fb..2cb1a6ab80 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -558,10 +558,6 @@ jobs: image: redis:alpine ports: - 11000:6379 - redis-socketio: - image: redis:alpine - ports: - - 12000:6379 mariadb: image: mariadb:10.6 env: From 98260b3c889a0ec4bd5628268e469eb2d6dc697e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 12 Jun 2023 13:53:08 +0530 Subject: [PATCH 29/29] fix(UX): show perm server messages on file uploader (#21331) --- frappe/permissions.py | 1 + .../js/frappe/file_uploader/FileUploader.vue | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/frappe/permissions.py b/frappe/permissions.py index 67ed972c32..b3380dacc0 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -97,6 +97,7 @@ def has_permission( if not perm: push_perm_check_log( _("User {0} does not have access to this document").format(frappe.bold(user)) + + f": {_(doc.doctype)} - {doc.name}" ) else: if ptype == "submit" and not cint(meta.is_submittable): diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index b17b13eb83..eba6bf1b2a 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -473,7 +473,20 @@ function upload_file(file, i) { } else if (xhr.status === 403) { file.failed = true; let response = JSON.parse(xhr.responseText); - file.error_message = `Not permitted. ${response._error_message || ''}`; + file.error_message = `Not permitted. ${response._error_message || ''}.`; + + try { + // Append server messages which are useful hint for perm issues + let server_messages = JSON.parse(response._server_messages); + + server_messages.forEach((m) => { + m = JSON.parse(m); + file.error_message += `\n ${m.message} ` + }) + } catch (e) { + console.warning("Failed to parse server message", e) + } + } else if (xhr.status === 413) { file.failed = true;