From 748e2ebb2b82621b05187c5769468a8d110e4240 Mon Sep 17 00:00:00 2001 From: hrwx Date: Mon, 15 Nov 2021 13:30:13 +0000 Subject: [PATCH 01/79] feat: multistep webforms --- frappe/public/js/frappe/web_form/web_form.js | 175 ++++++++++++++++++ frappe/website/doctype/web_form/web_form.json | 9 +- 2 files changed, 183 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/web_form/web_form.js b/frappe/public/js/frappe/web_form/web_form.js index 964a8ad0bb..8c975136dd 100644 --- a/frappe/public/js/frappe/web_form/web_form.js +++ b/frappe/public/js/frappe/web_form/web_form.js @@ -9,6 +9,7 @@ export default class WebForm extends frappe.ui.FieldGroup { frappe.web_form = this; frappe.web_form.events = {}; Object.assign(frappe.web_form.events, EventEmitterMixin); + this.current_section = 0; } prepare(web_form_doc, doc) { @@ -19,12 +20,16 @@ export default class WebForm extends frappe.ui.FieldGroup { make() { super.make(); + this.set_sections(); this.set_field_values(); + this.setup_listeners(); if (this.introduction_text) this.set_form_description(this.introduction_text); if (this.allow_print && !this.is_new) this.setup_print_button(); if (this.allow_delete && !this.is_new) this.setup_delete_button(); if (this.is_new) this.setup_cancel_button(); this.setup_primary_action(); + this.setup_previous_next_button(); + this.toggle_section(); $(".link-btn").remove(); // webform client script @@ -40,6 +45,79 @@ export default class WebForm extends frappe.ui.FieldGroup { }; } + setup_listeners() { + // Event listener for triggering Save/Next button for Multi Step Forms + // Do not use `on` event here since that can be used by user which will render this function useless + // setTimeout has 200ms delay so that all the base_control triggers for the fields have been run + let me = this; + + if (!me.is_multi_step_form) { + return; + } + + for (let field of $(".input-with-feedback")) { + $(field).change((e) => { + setTimeout(() => { + e.stopPropagation(); + me.toggle_buttons(); + }, 200); + }); + } + } + + set_sections() { + if (this.sections.length) return; + + this.sections = $(`.form-section`); + } + + setup_previous_next_button() { + let me = this; + + if (!me.is_multi_step_form) { + return; + } + + $('.web-form-footer').after(` + + `); + + $('.btn-previous').on('click', function () { + let is_validated = me.validate_section(); + + if (!is_validated) return; + + for (let idx = me.current_section; idx < me.sections.length; idx--) { + let is_empty = me.is_previous_section_empty(idx); + me.current_section = me.current_section > 0 ? me.current_section - 1 : me.current_section; + + if (!is_empty) { + break + } + } + me.toggle_section(); + }); + + $('.btn-next').on('click', function () { + let is_validated = me.validate_section(); + + if (!is_validated) return; + + for (let idx = me.current_section; idx < me.sections.length; idx++) { + let is_empty = me.is_next_section_empty(idx); + me.current_section = me.current_section < me.sections.length ? me.current_section + 1 : me.current_section; + + if (!is_empty) { + break + } + } + me.toggle_section(); + }); + } + set_field_values() { if (this.doc.name) this.set_values(this.doc); else return; @@ -104,6 +182,103 @@ export default class WebForm extends frappe.ui.FieldGroup { ); } + validate_section() { + if (this.allow_incomplete) return true; + + let fields = $(`.form-section:eq(${this.current_section}) .form-control`); + let errors = [] + + for (let field of fields) { + let fieldname = $(field).attr("data-fieldname"); + if (!fieldname) continue; + + field = this.fields_dict[fieldname]; + + if (field.get_value) { + let value = field.get_value(); + if (field.df.reqd && is_null(typeof value === 'string' ? strip_html(value) : value)) errors.push(__(field.df.label)); + + if (field.df.reqd && field.df.fieldtype === 'Text Editor' && is_null(strip_html(cstr(value)))) errors.push(__(field.df.label)); + } + } + + if (errors.length) { + frappe.msgprint({ + title: __('Missing Values Required'), + message: __('Following fields have missing values:') + + '

', + indicator: 'orange' + }); + return false; + } + + return true; + } + + toggle_section() { + if (!this.is_multi_step_form) return; + + this.toggle_previous_button(); + this.hide_sections(); + this.show_section(); + this.toggle_buttons(); + } + + toggle_buttons() { + for (let idx = this.current_section; idx < this.sections.length; idx++) { + if (this.is_next_section_empty(idx)) { + this.show_save_and_hide_next_button(); + } else { + this.show_next_and_hide_save_button(); + break; + } + } + } + + is_next_section_empty(section) { + if (section + 1 > this.sections.length) return true; + + let _section = $(`.form-section:eq(${section + 1})`); + let visible_controls = _section.find(".frappe-control:not(.hide-control)"); + + return !visible_controls.length ? true : false; + } + + is_previous_section_empty(section) { + if (section - 1 > this.sections.length) return true; + + let _section = $(`.form-section:eq(${section - 1})`); + let visible_controls = _section.find(".frappe-control:not(.hide-control)"); + + return !visible_controls.length ? true : false; + } + + show_save_and_hide_next_button() { + $('.btn-next').hide(); + $('.web-form-footer').show(); + } + + show_next_and_hide_save_button() { + $('.btn-next').show(); + $('.web-form-footer').hide(); + } + + toggle_previous_button() { + this.current_section == 0 ? $('.btn-previous').hide() : $('.btn-previous').show(); + } + + show_section() { + $(`.form-section:eq(${this.current_section})`).show(); + } + + hide_sections() { + for (let idx=0; idx < this.sections.length; idx++) { + if (idx !== this.current_section) { + $(`.form-section:eq(${idx})`).hide(); + } + } + } + save() { let is_new = this.is_new; if (this.validate && !this.validate()) { diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json index 9199183a65..260489fe25 100644 --- a/frappe/website/doctype/web_form/web_form.json +++ b/frappe/website/doctype/web_form/web_form.json @@ -11,6 +11,7 @@ "module", "column_break_4", "is_standard", + "is_multi_step_form", "published", "login_required", "route_to_success_link", @@ -355,13 +356,19 @@ "fieldname": "apply_document_permissions", "fieldtype": "Check", "label": "Apply Document Permissions" + }, + { + "default": "0", + "fieldname": "is_multi_step_form", + "fieldtype": "Check", + "label": "Is Multi Step Form" } ], "has_web_view": 1, "icon": "icon-edit", "is_published_field": "published", "links": [], - "modified": "2020-08-07 13:12:03.945686", + "modified": "2021-11-15 14:12:44.624573", "modified_by": "Administrator", "module": "Website", "name": "Web Form", From 1b906d8c78ab7a9f2a556a53d547952473bf32ae Mon Sep 17 00:00:00 2001 From: hrwx Date: Sun, 19 Dec 2021 00:58:17 +0000 Subject: [PATCH 02/79] feat: test cases for webform --- cypress/integration/web_form.js | 29 +++++++++++++++++++++++++++++ frappe/tests/ui_test_helpers.py | 7 +++++++ 2 files changed, 36 insertions(+) create mode 100644 cypress/integration/web_form.js diff --git a/cypress/integration/web_form.js b/cypress/integration/web_form.js new file mode 100644 index 0000000000..acdb4f6198 --- /dev/null +++ b/cypress/integration/web_form.js @@ -0,0 +1,29 @@ +context('Web Form', () => { + before(() => { + cy.login('Administrator', 'frappe') + }); + + it('Navigate and Submit a WebForm', () => { + cy.visit('/update-profile'); + cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); + cy.get('.web-form-actions .btn-primary').click(); + cy.wait(500); + cy.get('.modal.show > .modal-dialog').should('be.visible'); + }); + + it('Timeline should have submit and cancel activity information', () => { + cy.call('frappe.tests.ui_test_helpers.update_webform_to_multistep').then(r => { + cy.visit('/update-profile'); + cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); + cy.get('.btn-next').should('be.visible'); + cy.get('.web-form-footer .btn-primary').should('not.be.visible'); + cy.get('.btn-next').click(); + cy.get('.btn-previous').should('be.visible'); + cy.get('.btn-next').should('not.be.visible'); + cy.get('.web-form-footer .btn-primary').should('be.visible'); + cy.get('.web-form-actions .btn-primary').click(); + cy.wait(500); + cy.get('.modal.show > .modal-dialog').should('be.visible'); + }); + }); +}); diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 3e9b3519bc..075bb9fcec 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -244,3 +244,10 @@ def create_topic_and_reply(web_page): }) reply.save() + + +@frappe.whitelist() +def update_webform_to_multistep(): + doc = frappe.get_doc("Web Form", "edit-profile") + doc.is_multi_step_form = 1 + doc.save() From ae363d9df23a93a3a63e996f1fa13b0df71be8ea Mon Sep 17 00:00:00 2001 From: hrwx Date: Mon, 15 Nov 2021 13:30:13 +0000 Subject: [PATCH 03/79] feat: multistep webforms --- frappe/public/js/frappe/web_form/web_form.js | 175 ++++++++++++++++++ frappe/website/doctype/web_form/web_form.json | 9 +- 2 files changed, 183 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/web_form/web_form.js b/frappe/public/js/frappe/web_form/web_form.js index 964a8ad0bb..8c975136dd 100644 --- a/frappe/public/js/frappe/web_form/web_form.js +++ b/frappe/public/js/frappe/web_form/web_form.js @@ -9,6 +9,7 @@ export default class WebForm extends frappe.ui.FieldGroup { frappe.web_form = this; frappe.web_form.events = {}; Object.assign(frappe.web_form.events, EventEmitterMixin); + this.current_section = 0; } prepare(web_form_doc, doc) { @@ -19,12 +20,16 @@ export default class WebForm extends frappe.ui.FieldGroup { make() { super.make(); + this.set_sections(); this.set_field_values(); + this.setup_listeners(); if (this.introduction_text) this.set_form_description(this.introduction_text); if (this.allow_print && !this.is_new) this.setup_print_button(); if (this.allow_delete && !this.is_new) this.setup_delete_button(); if (this.is_new) this.setup_cancel_button(); this.setup_primary_action(); + this.setup_previous_next_button(); + this.toggle_section(); $(".link-btn").remove(); // webform client script @@ -40,6 +45,79 @@ export default class WebForm extends frappe.ui.FieldGroup { }; } + setup_listeners() { + // Event listener for triggering Save/Next button for Multi Step Forms + // Do not use `on` event here since that can be used by user which will render this function useless + // setTimeout has 200ms delay so that all the base_control triggers for the fields have been run + let me = this; + + if (!me.is_multi_step_form) { + return; + } + + for (let field of $(".input-with-feedback")) { + $(field).change((e) => { + setTimeout(() => { + e.stopPropagation(); + me.toggle_buttons(); + }, 200); + }); + } + } + + set_sections() { + if (this.sections.length) return; + + this.sections = $(`.form-section`); + } + + setup_previous_next_button() { + let me = this; + + if (!me.is_multi_step_form) { + return; + } + + $('.web-form-footer').after(` + + `); + + $('.btn-previous').on('click', function () { + let is_validated = me.validate_section(); + + if (!is_validated) return; + + for (let idx = me.current_section; idx < me.sections.length; idx--) { + let is_empty = me.is_previous_section_empty(idx); + me.current_section = me.current_section > 0 ? me.current_section - 1 : me.current_section; + + if (!is_empty) { + break + } + } + me.toggle_section(); + }); + + $('.btn-next').on('click', function () { + let is_validated = me.validate_section(); + + if (!is_validated) return; + + for (let idx = me.current_section; idx < me.sections.length; idx++) { + let is_empty = me.is_next_section_empty(idx); + me.current_section = me.current_section < me.sections.length ? me.current_section + 1 : me.current_section; + + if (!is_empty) { + break + } + } + me.toggle_section(); + }); + } + set_field_values() { if (this.doc.name) this.set_values(this.doc); else return; @@ -104,6 +182,103 @@ export default class WebForm extends frappe.ui.FieldGroup { ); } + validate_section() { + if (this.allow_incomplete) return true; + + let fields = $(`.form-section:eq(${this.current_section}) .form-control`); + let errors = [] + + for (let field of fields) { + let fieldname = $(field).attr("data-fieldname"); + if (!fieldname) continue; + + field = this.fields_dict[fieldname]; + + if (field.get_value) { + let value = field.get_value(); + if (field.df.reqd && is_null(typeof value === 'string' ? strip_html(value) : value)) errors.push(__(field.df.label)); + + if (field.df.reqd && field.df.fieldtype === 'Text Editor' && is_null(strip_html(cstr(value)))) errors.push(__(field.df.label)); + } + } + + if (errors.length) { + frappe.msgprint({ + title: __('Missing Values Required'), + message: __('Following fields have missing values:') + + '

  • ' + errors.join('
  • ') + '
', + indicator: 'orange' + }); + return false; + } + + return true; + } + + toggle_section() { + if (!this.is_multi_step_form) return; + + this.toggle_previous_button(); + this.hide_sections(); + this.show_section(); + this.toggle_buttons(); + } + + toggle_buttons() { + for (let idx = this.current_section; idx < this.sections.length; idx++) { + if (this.is_next_section_empty(idx)) { + this.show_save_and_hide_next_button(); + } else { + this.show_next_and_hide_save_button(); + break; + } + } + } + + is_next_section_empty(section) { + if (section + 1 > this.sections.length) return true; + + let _section = $(`.form-section:eq(${section + 1})`); + let visible_controls = _section.find(".frappe-control:not(.hide-control)"); + + return !visible_controls.length ? true : false; + } + + is_previous_section_empty(section) { + if (section - 1 > this.sections.length) return true; + + let _section = $(`.form-section:eq(${section - 1})`); + let visible_controls = _section.find(".frappe-control:not(.hide-control)"); + + return !visible_controls.length ? true : false; + } + + show_save_and_hide_next_button() { + $('.btn-next').hide(); + $('.web-form-footer').show(); + } + + show_next_and_hide_save_button() { + $('.btn-next').show(); + $('.web-form-footer').hide(); + } + + toggle_previous_button() { + this.current_section == 0 ? $('.btn-previous').hide() : $('.btn-previous').show(); + } + + show_section() { + $(`.form-section:eq(${this.current_section})`).show(); + } + + hide_sections() { + for (let idx=0; idx < this.sections.length; idx++) { + if (idx !== this.current_section) { + $(`.form-section:eq(${idx})`).hide(); + } + } + } + save() { let is_new = this.is_new; if (this.validate && !this.validate()) { diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json index 9199183a65..260489fe25 100644 --- a/frappe/website/doctype/web_form/web_form.json +++ b/frappe/website/doctype/web_form/web_form.json @@ -11,6 +11,7 @@ "module", "column_break_4", "is_standard", + "is_multi_step_form", "published", "login_required", "route_to_success_link", @@ -355,13 +356,19 @@ "fieldname": "apply_document_permissions", "fieldtype": "Check", "label": "Apply Document Permissions" + }, + { + "default": "0", + "fieldname": "is_multi_step_form", + "fieldtype": "Check", + "label": "Is Multi Step Form" } ], "has_web_view": 1, "icon": "icon-edit", "is_published_field": "published", "links": [], - "modified": "2020-08-07 13:12:03.945686", + "modified": "2021-11-15 14:12:44.624573", "modified_by": "Administrator", "module": "Website", "name": "Web Form", From b020b865ba16af925a450020dd7e3d90b9aae4b3 Mon Sep 17 00:00:00 2001 From: hrwx Date: Sun, 19 Dec 2021 00:58:17 +0000 Subject: [PATCH 04/79] feat: test cases for webform --- cypress/integration/web_form.js | 29 +++++++++++++++++++++++++++++ frappe/tests/ui_test_helpers.py | 7 +++++++ 2 files changed, 36 insertions(+) create mode 100644 cypress/integration/web_form.js diff --git a/cypress/integration/web_form.js b/cypress/integration/web_form.js new file mode 100644 index 0000000000..acdb4f6198 --- /dev/null +++ b/cypress/integration/web_form.js @@ -0,0 +1,29 @@ +context('Web Form', () => { + before(() => { + cy.login('Administrator', 'frappe') + }); + + it('Navigate and Submit a WebForm', () => { + cy.visit('/update-profile'); + cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); + cy.get('.web-form-actions .btn-primary').click(); + cy.wait(500); + cy.get('.modal.show > .modal-dialog').should('be.visible'); + }); + + it('Timeline should have submit and cancel activity information', () => { + cy.call('frappe.tests.ui_test_helpers.update_webform_to_multistep').then(r => { + cy.visit('/update-profile'); + cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); + cy.get('.btn-next').should('be.visible'); + cy.get('.web-form-footer .btn-primary').should('not.be.visible'); + cy.get('.btn-next').click(); + cy.get('.btn-previous').should('be.visible'); + cy.get('.btn-next').should('not.be.visible'); + cy.get('.web-form-footer .btn-primary').should('be.visible'); + cy.get('.web-form-actions .btn-primary').click(); + cy.wait(500); + cy.get('.modal.show > .modal-dialog').should('be.visible'); + }); + }); +}); diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 3e9b3519bc..075bb9fcec 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -244,3 +244,10 @@ def create_topic_and_reply(web_page): }) reply.save() + + +@frappe.whitelist() +def update_webform_to_multistep(): + doc = frappe.get_doc("Web Form", "edit-profile") + doc.is_multi_step_form = 1 + doc.save() From 28de383e8b056ce626970c077664822a8b269ba0 Mon Sep 17 00:00:00 2001 From: hrwx Date: Sun, 19 Dec 2021 12:33:15 +0000 Subject: [PATCH 05/79] chore: ignore for-direction in eslint --- frappe/public/js/frappe/web_form/web_form.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/web_form/web_form.js b/frappe/public/js/frappe/web_form/web_form.js index f71e681f85..517a93394d 100644 --- a/frappe/public/js/frappe/web_form/web_form.js +++ b/frappe/public/js/frappe/web_form/web_form.js @@ -90,7 +90,15 @@ export default class WebForm extends frappe.ui.FieldGroup { if (!is_validated) return; - for (let idx = me.current_section; idx < me.sections.length; idx--) { // eslint-disable-line no-console + /* + eslint cannot figure out if this is an infinite loop in backwards and + throws an error. Disabling for-direction just for this section. + for-direction doesnt throw an error if the values are hardcoded in the + reverse for-loop, but in this case its a dynamic loop. + https://eslint.org/docs/rules/for-direction + */ + /* eslint-disable for-direction */ + for (let idx = me.current_section; idx < me.sections.length; idx--) { let is_empty = me.is_previous_section_empty(idx); me.current_section = me.current_section > 0 ? me.current_section - 1 : me.current_section; @@ -98,6 +106,7 @@ export default class WebForm extends frappe.ui.FieldGroup { break; } } + /* eslint-enable for-direction */ me.toggle_section(); }); From 688829efa52a3dd940181ddbe2e7c89d6deb0920 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Tue, 28 Dec 2021 15:51:17 +0530 Subject: [PATCH 06/79] fix: time zone info in web form --- frappe/website/doctype/web_form/templates/web_form.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/website/doctype/web_form/templates/web_form.html b/frappe/website/doctype/web_form/templates/web_form.html index f511b3c27d..743c094314 100644 --- a/frappe/website/doctype/web_form/templates/web_form.html +++ b/frappe/website/doctype/web_form/templates/web_form.html @@ -78,6 +78,10 @@ frappe.boot = { sysdefaults: { float_precision: parseInt("{{ frappe.get_system_settings('float_precision') or 3 }}"), date_format: "{{ frappe.get_system_settings('date_format') or 'yyyy-mm-dd' }}", + }, + time_zone: { + system: "{{ frappe.utils.get_time_zone() }}", + user: "{{ frappe.db.get_value('User', frappe.session.user, 'time_zone') or frappe.utils.get_time_zone() }}" } }; // for backward compatibility of some libs From 409ebfcda5624b6219839507f5da6469b046b3c5 Mon Sep 17 00:00:00 2001 From: hrwx Date: Wed, 29 Dec 2021 01:52:06 +0530 Subject: [PATCH 07/79] fix: show invalid value error --- frappe/public/js/frappe/web_form/web_form.js | 28 +++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/frappe/public/js/frappe/web_form/web_form.js b/frappe/public/js/frappe/web_form/web_form.js index 517a93394d..273aa08b45 100644 --- a/frappe/public/js/frappe/web_form/web_form.js +++ b/frappe/public/js/frappe/web_form/web_form.js @@ -196,6 +196,7 @@ export default class WebForm extends frappe.ui.FieldGroup { let fields = $(`.form-section:eq(${this.current_section}) .form-control`); let errors = []; + let invalid_values = []; for (let field of fields) { let fieldname = $(field).attr("data-fieldname"); @@ -208,20 +209,29 @@ export default class WebForm extends frappe.ui.FieldGroup { if (field.df.reqd && is_null(typeof value === 'string' ? strip_html(value) : value)) errors.push(__(field.df.label)); if (field.df.reqd && field.df.fieldtype === 'Text Editor' && is_null(strip_html(cstr(value)))) errors.push(__(field.df.label)); + + if (field.df.invalid) invalid_values.push(__(field.df.label)); } } - if (errors.length) { - frappe.msgprint({ - title: __('Missing Values Required'), - message: __('Following fields have missing values:') + - '

  • ' + errors.join('
  • ') + '
', - indicator: 'orange' - }); - return false; + let message = ''; + if (invalid_values.length) { + message += __('Invalid values for fields:') + '

  • ' + invalid_values.join('
  • ') + '
'; } - return true; + if (errors.length) { + message += __('Mandatory fields required:') + '

  • ' + errors.join('
  • ') + '
'; + } + + if (invalid_values.length || errors.length) { + frappe.msgprint({ + title: __('Error'), + message: message, + indicator: 'orange' + }); + } + + return !(errors.length || invalid_values.length); } toggle_section() { From 950f60ef69ad1ed8b5567619b511a589503de8e6 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 29 Dec 2021 22:41:24 +0530 Subject: [PATCH 08/79] fix: sr no column alignment --- frappe/public/js/frappe/form/grid_row.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 96e502663d..a40f428969 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -196,7 +196,7 @@ export default class GridRow { // REDESIGN-TODO: Make translation contextual, this No is Number var txt = (this.doc ? this.doc.idx : __("No.")); this.row_index = $( - `
+ `
${this.row_check_html}
`) .appendTo(this.row) From 39c0577fc5e59cdaaefd8c0654732f8b86fa6bc8 Mon Sep 17 00:00:00 2001 From: hrwx Date: Thu, 30 Dec 2021 02:09:51 +0530 Subject: [PATCH 09/79] chore: update cypress test cases --- cypress/integration/web_form.js | 2 +- frappe/tests/ui_test_helpers.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cypress/integration/web_form.js b/cypress/integration/web_form.js index 5f5ac203cf..8346c96313 100644 --- a/cypress/integration/web_form.js +++ b/cypress/integration/web_form.js @@ -13,7 +13,7 @@ context('Web Form', () => { it('Navigate and Submit a MultiStep WebForm', () => { cy.call('frappe.tests.ui_test_helpers.update_webform_to_multistep').then(() => { - cy.visit('/update-profile'); + cy.visit('/update-profile-duplicate'); cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); cy.get('.btn-next').should('be.visible'); cy.get('.web-form-footer .btn-primary').should('not.be.visible'); diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 075bb9fcec..79868b0b76 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -249,5 +249,9 @@ def create_topic_and_reply(web_page): @frappe.whitelist() def update_webform_to_multistep(): doc = frappe.get_doc("Web Form", "edit-profile") - doc.is_multi_step_form = 1 - doc.save() + _doc = frappe.copy_doc(doc) + _doc.is_multi_step_form = 1 + _doc.title = "update-profile-duplicate" + _doc.route = "update-profile-duplicate" + _doc.is_standard = False + _doc.save() From 24830298b03272655e916af620d496f458a69c6b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 8 Nov 2021 18:27:29 +0530 Subject: [PATCH 10/79] fix: Make owner, creation fields constant for docs --- frappe/model/meta.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 252c463d3d..a483f3f2d6 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -67,6 +67,10 @@ class Meta(Document): _metaclass = True default_fields = list(default_fields)[1:] special_doctypes = ("DocField", "DocPerm", "DocType", "Module Def", 'DocType Action', 'DocType Link', 'DocType State') + standard_set_once_fields = [ + frappe._dict(fieldname="creation", fieldtype="Datetime"), + frappe._dict(fieldname="owner", fieldtype="Data"), + ] def __init__(self, doctype): self._fields = {} @@ -154,6 +158,12 @@ class Meta(Document): '''Return fields with `set_only_once` set''' if not hasattr(self, "_set_only_once_fields"): self._set_only_once_fields = self.get("fields", {"set_only_once": 1}) + fieldnames = [d.fieldname for d in self._set_only_once_fields] + + for df in self.standard_set_once_fields: + if df.fieldname not in fieldnames: + self._set_only_once_fields.append(df) + return self._set_only_once_fields def get_table_fields(self): From 62499422a85a38b6dfbc16080674149d9108c852 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 8 Nov 2021 18:28:36 +0530 Subject: [PATCH 11/79] style: Make fieldname bold in user message --- frappe/model/document.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 1f079feedc..891ad1d8de 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -562,8 +562,12 @@ class Document(BaseDocument): fail = value != original_value if fail: - frappe.throw(_("Value cannot be changed for {0}").format(self.meta.get_label(field.fieldname)), - frappe.CannotChangeConstantError) + frappe.throw( + _("Value cannot be changed for {0}").format( + frappe.bold(self.meta.get_label(field.fieldname)) + ), + exc=frappe.CannotChangeConstantError + ) return False From 568d9b94ed4b88e812a65fc2c1c001df89092152 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 8 Nov 2021 19:00:17 +0530 Subject: [PATCH 12/79] test: Use db_set to change creation, status --- .../test_personal_data_deletion_request.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py index 8fc8f38512..27dcfe5858 100644 --- a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py @@ -50,11 +50,10 @@ class TestPersonalDataDeletionRequest(unittest.TestCase): def test_unverified_record_removal(self): date_time_obj = datetime.strptime( self.delete_request.creation, "%Y-%m-%d %H:%M:%S.%f" - ) - date_time_obj += timedelta(days=-7) - self.delete_request.creation = date_time_obj - self.status = "Pending Verification" - self.delete_request.save() + ) + timedelta(days=-7) + self.delete_request.db_set("creation", date_time_obj) + self.delete_request.db_set("status", "Pending Verification") + remove_unverified_record() self.assertFalse( frappe.db.exists("Personal Data Deletion Request", self.delete_request.name) From 272cea49405cb5cbf3cc6be04659358d52c4968b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 9 Nov 2021 01:17:26 +0530 Subject: [PATCH 13/79] test: Add test for owner, creation constants --- frappe/tests/test_permissions.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/frappe/tests/test_permissions.py b/frappe/tests/test_permissions.py index b4e7db9956..d83f2969ba 100644 --- a/frappe/tests/test_permissions.py +++ b/frappe/tests/test_permissions.py @@ -12,6 +12,7 @@ from frappe.core.page.permission_manager.permission_manager import update, reset from frappe.test_runner import make_test_records_for_doctype from frappe.core.doctype.user_permission.user_permission import clear_user_permissions from frappe.desk.form.load import getdoc +from frappe.utils.data import now_datetime test_dependencies = ['Blogger', 'Blog Post', "User", "Contact", "Salutation"] @@ -197,6 +198,17 @@ class TestPermissions(unittest.TestCase): doc = frappe.get_doc("Blog Post", "-test-blog-post") self.assertTrue(doc.has_permission("read")) + def test_dont_change_standard_constants(self): + # check that Document.creation cannot be changed + user = frappe.get_doc("User", frappe.session.user) + user.creation = now_datetime() + self.assertRaises(frappe.CannotChangeConstantError, user.save) + + # check that Document.owner cannot be changed + user.reload() + user.owner = frappe.db.get_value("User", {"name": ("!=", user.name)}) + self.assertRaises(frappe.CannotChangeConstantError, user.save) + def test_set_only_once(self): blog_post = frappe.get_meta("Blog Post") doc = frappe.get_doc("Blog Post", "-test-blog-post-1") From 881f3ad8c1bb7c88a275e8504ae183363985f674 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Dec 2021 10:35:49 +0530 Subject: [PATCH 14/79] fix: Set owner & creation if new Document via https://github.com/frappe/frappe/pull/14918/commits/a323d624eb3a59942aaac70ec95412140e70b7cb --- frappe/model/document.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 891ad1d8de..16a2d35290 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -396,6 +396,7 @@ class Document(BaseDocument): "parenttype": self.doctype, "parentfield": fieldname }) + def get_doc_before_save(self): return getattr(self, '_doc_before_save', None) @@ -468,9 +469,11 @@ class Document(BaseDocument): self._original_modified = self.modified self.modified = now() self.modified_by = frappe.session.user - if not self.creation: + + # We'd probably want the creation and owner to be set via API + # or Data import at some point, that'd have to be handled here + if self.is_new(): self.creation = self.modified - if not self.owner: self.owner = self.modified_by for d in self.get_all_children(): From ecb0cd41398d169a8eb849420f992cf086fc6872 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Dec 2021 10:37:12 +0530 Subject: [PATCH 15/79] test: Add test for disallowing setting tandard fields Via https://github.com/frappe/frappe/pull/14918/commits/db008020f69408a1ce278a311645fb861b5a78d7 --- frappe/tests/test_permissions.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/frappe/tests/test_permissions.py b/frappe/tests/test_permissions.py index d83f2969ba..fdff4d103e 100644 --- a/frappe/tests/test_permissions.py +++ b/frappe/tests/test_permissions.py @@ -198,6 +198,21 @@ class TestPermissions(unittest.TestCase): doc = frappe.get_doc("Blog Post", "-test-blog-post") self.assertTrue(doc.has_permission("read")) + def test_set_standard_fields_manually(self): + # check that creation and owner cannot be set manually + from datetime import timedelta + + fake_creation = now_datetime() + timedelta(days=-7) + fake_owner = frappe.db.get_value("User", {"name": ("!=", frappe.session.user)}) + + d = frappe.new_doc("ToDo") + d.description = "ToDo created via test_set_standard_fields_manually" + d.creation = fake_creation + d.owner = fake_owner + d.save() + self.assertNotEqual(d.creation, fake_creation) + self.assertNotEqual(d.owner, fake_owner) + def test_dont_change_standard_constants(self): # check that Document.creation cannot be changed user = frappe.get_doc("User", frappe.session.user) From 063cc293a950392ff2002c6d9b823f233793f69f Mon Sep 17 00:00:00 2001 From: Saif Ur Rehman Date: Fri, 31 Dec 2021 13:30:02 +0500 Subject: [PATCH 16/79] fix(Migrate): add unique and index for new columns --- frappe/database/schema.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/database/schema.py b/frappe/database/schema.py index ce9fcb4147..10582eff8f 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -206,6 +206,12 @@ class DbColumn: if not current_def: self.fieldname = validate_column_name(self.fieldname) self.table.add_column.append(self) + + if column_type not in ('text', 'longtext'): + if self.unique: + self.table.add_unique.append(self) + if self.set_index: + self.table.add_index.append(self) return # type From 289d7e7afa8c922d9069579d3a898ba98b529a9b Mon Sep 17 00:00:00 2001 From: Pruthvi Patel Date: Sat, 1 Jan 2022 17:49:59 +0530 Subject: [PATCH 17/79] fix: clear cache when filters passed as dn param in `set_value` --- frappe/database/database.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 7c147cd1d0..165dc729ed 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -699,6 +699,8 @@ class Database(object): self.sql("""update `tab{0}` set {1} where name=%(name)s""".format(dt, ', '.join(set_values)), values, debug=debug) + + frappe.clear_document_cache(dt, values['name']) else: # for singles keys = list(to_update) @@ -711,10 +713,11 @@ class Database(object): self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''', (dt, key, value), debug=debug) + frappe.clear_document_cache(dt, dn) + if dt in self.value_cache: del self.value_cache[dt] - frappe.clear_document_cache(dt, dn) @staticmethod def set(doc, field, val): From b5e2f78a883d13e5acd477130aab7dc59e1fc0c2 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Jan 2022 13:36:32 +0530 Subject: [PATCH 18/79] fix: Start week from Sunday instead of Monday - Like we do in client-side --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 206f0eac64..7a65a6f18a 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -246,7 +246,7 @@ def get_quarter_start(dt, as_str=False): def get_first_day_of_week(dt, as_str=False): dt = getdate(dt) - date = dt - datetime.timedelta(days=dt.weekday()) + date = dt - datetime.timedelta(days=(dt.weekday() + 1) % 7) return date.strftime(DATE_FORMAT) if as_str else date def get_year_start(dt, as_str=False): From 85360ef4c0c4204cf79c6248e122924993310f19 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Jan 2022 13:37:11 +0530 Subject: [PATCH 19/79] test: Add cases to check date utilities --- frappe/tests/test_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 599a638ce2..67586719c8 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -306,3 +306,16 @@ class TestDiffUtils(unittest.TestCase): diff = get_version_diff(old_version, latest_version) self.assertIn('-2;', diff) self.assertIn('+42;', diff) + +class TestDateUtils(unittest.TestCase): + def test_first_day_of_week(self): + self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-25"), + frappe.utils.getdate("2020-12-20")) + self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-21"), + frappe.utils.getdate("2020-12-20")) + + def test_last_day_of_week(self): + self.assertEqual(frappe.utils.get_last_day_of_week("2020-12-24"), + frappe.utils.getdate("2020-12-26")) + self.assertEqual(frappe.utils.get_last_day_of_week("2020-12-28"), + frappe.utils.getdate("2021-01-02")) \ No newline at end of file From e3fc400fd4d9385995d4bf13d0bf586f0e4e675b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 3 Jan 2022 19:01:01 +0530 Subject: [PATCH 20/79] refactor(minor): Override clear_cache instead of separate hooks Clear cached maps via Document.clear_cache --- .../assignment_rule/assignment_rule.py | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index a3e27d4da5..5a5581f2bc 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -3,14 +3,14 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document -from frappe.desk.form import assign_to -import frappe.cache_manager from frappe import _ +from frappe.cache_manager import clear_doctype_map, get_doctype_map +from frappe.desk.form import assign_to from frappe.model import log_types +from frappe.model.document import Document + class AssignmentRule(Document): - def validate(self): assignment_days = self.get_assignment_days() if not len(set(assignment_days)) == len(assignment_days): @@ -19,14 +19,10 @@ class AssignmentRule(Document): if self.document_type == 'ToDo': frappe.throw(_('Assignment Rule is not allowed on {0} document type').format(frappe.bold("ToDo"))) - def on_update(self): - clear_assignment_rule_cache(self) - - def after_rename(self, old, new, merge): - clear_assignment_rule_cache(self) - - def on_trash(self): - clear_assignment_rule_cache(self) + def clear_cache(self): + super().clear_cache() + clear_doctype_map(self.doctype, self.document_type) + clear_doctype_map(self.doctype, f"due_date_rules_for_{self.document_type}") def apply_unassign(self, doc, assignments): if (self.unassign_condition and @@ -35,7 +31,6 @@ class AssignmentRule(Document): return False - def apply_assign(self, doc): if self.safe_eval('assign_condition', doc): return self.do_assignment(doc) @@ -296,6 +291,3 @@ def get_repeated(values): diff.append(str(value)) return " ".join(diff) -def clear_assignment_rule_cache(rule): - frappe.cache_manager.clear_doctype_map('Assignment Rule', rule.document_type) - frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type) From f72c445d41ad25f019ad713170885530b5a93e59 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 3 Jan 2022 19:02:41 +0530 Subject: [PATCH 21/79] fix: Clear Document cache on rename, delete --- frappe/model/delete_doc.py | 3 ++- frappe/model/rename_doc.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index ac976e976c..2fddcf9e33 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -117,7 +117,8 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa doctype=doc.doctype, name=doc.name, is_async=False if frappe.flags.in_test else True) - + # clear cache for Document + doc.clear_cache() # delete global search entry delete_for_document(doc) # delete tag link entry diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 651153876a..f84242626b 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -110,6 +110,7 @@ def rename_doc( if merge: frappe.delete_doc(doctype, old) + new_doc.clear_cache() frappe.clear_cache() if rebuild_search: frappe.enqueue('frappe.utils.global_search.rebuild_for_doctype', doctype=doctype) From 3a384565836ffb961e1db32189035582961f0b87 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 3 Jan 2022 19:03:56 +0530 Subject: [PATCH 22/79] refactor(minor): Assignment Rule * Simplified and refactored logic * Added type hints for where it gets really unbearable :') * Minor perf improvements & better API usages --- .../assignment_rule/assignment_rule.py | 201 ++++++++++++------ 1 file changed, 131 insertions(+), 70 deletions(-) diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index 5a5581f2bc..724c453972 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and contributors +# Copyright (c) 2022, Frappe Technologies and contributors # License: MIT. See LICENSE +from typing import Dict, Iterable, List + import frappe from frappe import _ from frappe.cache_manager import clear_doctype_map, get_doctype_map @@ -12,18 +13,36 @@ from frappe.model.document import Document class AssignmentRule(Document): def validate(self): - assignment_days = self.get_assignment_days() - if not len(set(assignment_days)) == len(assignment_days): - repeated_days = get_repeated(assignment_days) - frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days))) - if self.document_type == 'ToDo': - frappe.throw(_('Assignment Rule is not allowed on {0} document type').format(frappe.bold("ToDo"))) + self.validate_document_types() + self.validate_assignment_days() def clear_cache(self): super().clear_cache() clear_doctype_map(self.doctype, self.document_type) clear_doctype_map(self.doctype, f"due_date_rules_for_{self.document_type}") + def validate_document_types(self): + if self.document_type == "ToDo": + frappe.throw( + _('Assignment Rule is not allowed on {0} document type').format( + frappe.bold("ToDo") + ) + ) + + def validate_assignment_days(self): + assignment_days = self.get_assignment_days() + + if len(set(assignment_days)) != len(assignment_days): + repeated_days = get_repeated(assignment_days) + plural = "s" if len(repeated_days) > 1 else "" + + frappe.throw( + _("Assignment Day{0} {1} has been repeated.").format( + plural, + frappe.bold(", ".join(repeated_days)) + ) + ) + def apply_unassign(self, doc, assignments): if (self.unassign_condition and self.name in [d.assignment_rule for d in assignments]): @@ -136,65 +155,68 @@ class AssignmentRule(Document): def is_rule_not_applicable_today(self): today = frappe.flags.assignment_day or frappe.utils.get_weekday() assignment_days = self.get_assignment_days() - if assignment_days and not today in assignment_days: - return True + return assignment_days and today not in assignment_days - return False -def get_assignments(doc): +def get_assignments(doc) -> List[Dict]: return frappe.get_all('ToDo', fields = ['name', 'assignment_rule'], filters = dict( reference_type = doc.get('doctype'), reference_name = doc.get('name'), status = ('!=', 'Cancelled') - ), limit = 5) + ), limit=5) + @frappe.whitelist() def bulk_apply(doctype, docnames): - import json - docnames = json.loads(docnames) - + docnames = frappe.parse_json(docnames) background = len(docnames) > 5 + for name in docnames: if background: frappe.enqueue('frappe.automation.doctype.assignment_rule.assignment_rule.apply', doc=None, doctype=doctype, name=name) else: - apply(None, doctype=doctype, name=name) + apply(doctype=doctype, name=name) + def reopen_closed_assignment(doc): - todo_list = frappe.db.get_all('ToDo', filters = dict( - reference_type = doc.doctype, - reference_name = doc.name, - status = 'Closed' - )) - if not todo_list: - return False + todo_list = frappe.get_all("ToDo", filters={ + "reference_type": doc.doctype, + "reference_name": doc.name, + "status": "Closed", + }, pluck="name") + for todo in todo_list: - todo_doc = frappe.get_doc('ToDo', todo.name) + todo_doc = frappe.get_doc('ToDo', todo) todo_doc.status = 'Open' todo_doc.save(ignore_permissions=True) - return True -def apply(doc, method=None, doctype=None, name=None): - if not doctype: - doctype = doc.doctype + return bool(todo_list) - if (frappe.flags.in_patch + +def apply(doc=None, method=None, doctype=None, name=None): + doctype = doctype or doc.doctype + + skip_assignment_rules = ( + frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_setup_wizard - or doctype in log_types): + or doctype in log_types + ) + + if skip_assignment_rules: return if not doc and doctype and name: doc = frappe.get_doc(doctype, name) - assignment_rules = frappe.cache_manager.get_doctype_map('Assignment Rule', doc.doctype, dict( - document_type = doc.doctype, disabled = 0), order_by = 'priority desc') - - assignment_rule_docs = [] + assignment_rules = get_doctype_map("Assignment Rule", doc.doctype, filters={ + "document_type": doc.doctype, "disabled": 0 + }, order_by="priority desc") # multiple auto assigns - for d in assignment_rules: - assignment_rule_docs.append(frappe.get_cached_doc('Assignment Rule', d.get('name'))) + assignment_rule_docs: List[AssignmentRule] = [ + frappe.get_cached_doc("Assignment Rule", d.get('name')) for d in assignment_rules + ] if not assignment_rule_docs: return @@ -230,6 +252,7 @@ def apply(doc, method=None, doctype=None, name=None): # apply close rule only if assignments exists assignments = get_assignments(doc) + if assignments: for assignment_rule in assignment_rule_docs: if assignment_rule.is_rule_not_applicable_today(): @@ -237,38 +260,74 @@ def apply(doc, method=None, doctype=None, name=None): if not new_apply: # only reopen if close condition is not satisfied - if not assignment_rule.safe_eval('close_condition', doc): - reopen = reopen_closed_assignment(doc) - if reopen: + to_close_todos = assignment_rule.safe_eval('close_condition', doc) + + if to_close_todos: + # close todo status + todos_to_close = frappe.get_all("ToDo", filters={ + "reference_type": doc.doctype, + "reference_name": doc.name, + }, pluck="name") + + for todo in todos_to_close: + _todo = frappe.get_doc("ToDo", todo) + _todo.status = "Closed" + _todo.save() + break + + else: + reopened = reopen_closed_assignment(doc) + if reopened: break + + # print(f"Rule:{assignment_rule}\nDoc: {doc}\nReOpened: {reopened}") + assignment_rule.close_assignments(doc) + def update_due_date(doc, state=None): - # called from hook - if (frappe.flags.in_patch - or frappe.flags.in_install - or frappe.flags.in_migrate + """Run on_update on every Document (via hooks.py) + """ + skip_document_update = ( + frappe.flags.in_migrate + or frappe.flags.in_patch or frappe.flags.in_import - or frappe.flags.in_setup_wizard): + or frappe.flags.in_setup_wizard + or frappe.flags.in_install + ) + + if skip_document_update: return - assignment_rules = frappe.cache_manager.get_doctype_map('Assignment Rule', 'due_date_rules_for_' + doc.doctype, dict( - document_type = doc.doctype, - disabled = 0, - due_date_based_on = ['is', 'set'] - )) + + assignment_rules = get_doctype_map( + doctype="Assignment Rule", + name=f"due_date_rules_for_{doc.doctype}", + filters={ + "due_date_based_on": ["is", "set"], + "document_type": doc.doctype, + "disabled": 0, + } + ) + for rule in assignment_rules: - rule_doc = frappe.get_cached_doc('Assignment Rule', rule.get('name')) + rule_doc = frappe.get_cached_doc("Assignment Rule", rule.get("name")) due_date_field = rule_doc.due_date_based_on - if doc.meta.has_field(due_date_field) and \ - doc.has_value_changed(due_date_field) and rule.get('name'): - assignment_todos = frappe.get_all('ToDo', { - 'assignment_rule': rule.get('name'), - 'status': 'Open', - 'reference_type': doc.doctype, - 'reference_name': doc.name - }) + field_updated = ( + doc.meta.has_field(due_date_field) + and doc.has_value_changed(due_date_field) + and rule.get("name") + ) + + if field_updated: + assignment_todos = frappe.get_all("ToDo", filters={ + "assignment_rule": rule.get("name"), + "reference_type": doc.doctype, + "reference_name": doc.name, + "status": "Open", + }, pluck="name") + for todo in assignment_todos: - todo_doc = frappe.get_doc('ToDo', todo.name) + todo_doc = frappe.get_doc('ToDo', todo) todo_doc.date = doc.get(due_date_field) todo_doc.flags.updater_reference = { 'doctype': 'Assignment Rule', @@ -277,17 +336,19 @@ def update_due_date(doc, state=None): } todo_doc.save(ignore_permissions=True) -def get_assignment_rules(): - return [d.document_type for d in frappe.db.get_all('Assignment Rule', fields=['document_type'], filters=dict(disabled = 0))] -def get_repeated(values): - unique_list = [] - diff = [] +def get_assignment_rules() -> List[str]: + return frappe.get_all("Assignment Rule", filters={"disabled": 0}, pluck="document_type") + + +def get_repeated(values: Iterable) -> List: + unique = set() + repeated = set() + for value in values: - if value not in unique_list: - unique_list.append(str(value)) + if value in unique: + repeated.add(value) else: - if value not in diff: - diff.append(str(value)) - return " ".join(diff) + unique.add(value) + return [str(x) for x in repeated] From f7b59829196a3e0594d2225b37eba8874d129bf4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 3 Jan 2022 19:09:42 +0530 Subject: [PATCH 23/79] test(assign_rule): Setup and teardown for local testing --- .../assignment_rule/test_assignment_rule.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py index 1c9e177f94..f20253b779 100644 --- a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py @@ -1,12 +1,22 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and Contributors +# Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe + import unittest -from frappe.utils import random_string + +import frappe from frappe.test_runner import make_test_records +from frappe.utils import random_string + class TestAutoAssign(unittest.TestCase): + @classmethod + def setUpClass(cls): + frappe.db.delete("Assignment Rule") + + @classmethod + def tearDownClass(cls): + frappe.db.rollback() + def setUp(self): make_test_records("User") days = [ @@ -129,7 +139,7 @@ class TestAutoAssign(unittest.TestCase): reference_type = 'Note', reference_name = note.name, status = 'Open' - ))[0] + ), limit=1)[0] todo = frappe.get_doc('ToDo', todo['name']) self.assertEqual(todo.owner, 'test@example.com') @@ -151,7 +161,7 @@ class TestAutoAssign(unittest.TestCase): reference_type = 'Note', reference_name = note.name, status = 'Open' - ))[0] + ), limit=1)[0] todo = frappe.get_doc('ToDo', todo['name']) self.assertEqual(todo.owner, 'test@example.com') From 322a777b597990b8bfd66bda2531a4a97aeb35b9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 3 Jan 2022 19:10:28 +0530 Subject: [PATCH 24/79] fix(ux): Auto Repeat's validation message has grammatical fixes --- frappe/automation/doctype/auto_repeat/auto_repeat.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index 5ab6c86c00..0277b8e402 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -96,7 +96,15 @@ class AutoRepeat(Document): auto_repeat_days = self.get_auto_repeat_days() if not len(set(auto_repeat_days)) == len(auto_repeat_days): repeated_days = get_repeated(auto_repeat_days) - frappe.throw(_('Auto Repeat Day {0} has been repeated.').format(frappe.bold(repeated_days))) + plural = "s" if len(repeated_days) > 1 else "" + + frappe.throw( + _("Auto Repeat Day{0} {1} has been repeated.").format( + plural, + frappe.bold(", ".join(repeated_days)) + ) + ) + def update_auto_repeat_id(self): #check if document is already on auto repeat From d603aaf6018d5156063713f35a699f2e196854ff Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jan 2022 23:05:33 +0530 Subject: [PATCH 25/79] fix: define is_syntax_error for postgres --- frappe/database/postgres/database.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 008635b1b3..c163b72d14 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -138,6 +138,10 @@ class PostgresDatabase(Database): # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError return isinstance(e, psycopg2.extensions.QueryCanceledError) + @staticmethod + def is_syntax_error(e): + return isinstance(e, psycopg2.errors.SyntaxError) + @staticmethod def is_table_missing(e): return getattr(e, 'pgcode', None) == '42P01' From 0fddbafd09f85a0754c40de534689d17fa359cbd Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 3 Jan 2022 17:42:11 +0530 Subject: [PATCH 26/79] fix(postgres)!: dont silently rollback on db exception --- frappe/database/database.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index a157343be6..856c85416f 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -164,10 +164,7 @@ class Database(object): frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2))) except Exception as e: - if frappe.conf.db_type == 'postgres': - self.rollback() - - elif self.is_syntax_error(e): + if self.is_syntax_error(e): # only for mariadb frappe.errprint('Syntax error in query:') frappe.errprint(query) @@ -178,6 +175,9 @@ class Database(object): elif self.is_timedout(e): raise frappe.QueryTimeoutError(e) + elif frappe.conf.db_type == 'postgres': + raise + if ignore_ddl and (self.is_missing_column(e) or self.is_missing_table(e) or self.cant_drop_field_or_key(e)): pass else: From 42e1c15c18f72e447f1b7b5e4227963abbc90249 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 13:00:31 +0530 Subject: [PATCH 27/79] feat: Add setting to configure the day on which week starts --- .../system_settings/system_settings.json | 13 ++++++-- .../doctype/system_console/system_console.js | 4 +++ frappe/public/js/frappe/desk.js | 19 ++++++++---- frappe/public/js/frappe/utils/datetime.js | 5 ++++ frappe/utils/data.py | 30 ++++++++++++++++++- 5 files changed, 63 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 3e04643256..0af8f2e4be 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -17,10 +17,11 @@ "date_and_number_format", "date_format", "time_format", - "column_break_7", "number_format", + "column_break_7", "float_precision", "currency_precision", + "week_starts_on", "sec_backup_limit", "backup_limit", "encrypt_backup", @@ -477,12 +478,19 @@ "fieldname": "disable_system_update_notification", "fieldtype": "Check", "label": "Disable System Update Notification" + }, + { + "default": "Sunday", + "fieldname": "week_starts_on", + "fieldtype": "Select", + "label": "Week Starts On", + "options": "Sunday\nMonday\nTuesday\nWednesday\nThursday\nFriday\nSaturday" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2021-11-29 18:09:53.601629", + "modified": "2022-01-04 11:28:34.881192", "modified_by": "Administrator", "module": "Core", "name": "System Settings", @@ -499,5 +507,6 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "ASC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js index 0fe3932671..06f04e8b32 100644 --- a/frappe/desk/doctype/system_console/system_console.js +++ b/frappe/desk/doctype/system_console/system_console.js @@ -100,5 +100,9 @@ frappe.ui.form.on('System Console', { ${rows}`); }); + }, + + week_starts_on() { + frappe.app.setup_moment() } }); diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 64767e1232..f9a9931c4c 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -275,11 +275,7 @@ frappe.Application = class Application { this.set_globals(); this.sync_pages(); frappe.router.setup(); - moment.locale("en"); - moment.user_utc_offset = moment().utcOffset(); - if(frappe.boot.timezone_info) { - moment.tz.add(frappe.boot.timezone_info); - } + this.setup_moment(); if(frappe.boot.print_css) { frappe.dom.set_style(frappe.boot.print_css, "print-style"); } @@ -628,6 +624,19 @@ frappe.Application = class Application { } }); } + + setup_moment() { + moment.updateLocale('en', { + week : { + dow : frappe.datetime.get_week_starts_on_index(), + } + }); + moment.locale("en"); + moment.user_utc_offset = moment().utcOffset(); + if(frappe.boot.timezone_info) { + moment.tz.add(frappe.boot.timezone_info); + } + } } frappe.get_module = function(m, default_module) { diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 7bb6076b72..09e44d917d 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -254,6 +254,11 @@ $.extend(frappe.datetime, { ], true).isValid(); }, + get_week_starts_on_index() { + const week_starts_on = frappe.sys_defaults.week_starts_on || "Sunday"; + return moment.weekdays().indexOf(week_starts_on); + } + }); // Proxy for dateutil and get_today diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 7a65a6f18a..9f9391b1fc 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -11,11 +11,26 @@ from code import compile_command from urllib.parse import quote, urljoin from frappe.desk.utils import slug from click import secho +from enum import Enum DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S.%f" DATETIME_FORMAT = DATE_FORMAT + " " + TIME_FORMAT +class Weekday(Enum): + Sunday = 0 + Monday = 1 + Tuesday = 2 + Wednesday = 3 + Thursday = 4 + Friday = 5 + Saturday = 6 + +def get_week_starts_on(): + return frappe.get_system_settings('week_starts_on') or "Sunday" + +def get_start_of_week_index(): + return Weekday[get_week_starts_on()].value def is_invalid_date_string(date_string): # dateutil parser does not agree with dates like "0001-01-01" or "0000-00-00" @@ -246,9 +261,22 @@ def get_quarter_start(dt, as_str=False): def get_first_day_of_week(dt, as_str=False): dt = getdate(dt) - date = dt - datetime.timedelta(days=(dt.weekday() + 1) % 7) + date = dt - datetime.timedelta(days=get_week_start_offset_days(dt)) return date.strftime(DATE_FORMAT) if as_str else date +def get_week_start_offset_days(dt): + current_day_index = get_normalized_weekday_index(dt) + start_of_week_index = get_start_of_week_index() + + if current_day_index >= start_of_week_index: + return current_day_index - start_of_week_index + else: + return 7 - (start_of_week_index - current_day_index) + +def get_normalized_weekday_index(dt): + # starts Sunday with 0 + return (dt.weekday() + 1) % 7 + def get_year_start(dt, as_str=False): dt = getdate(dt) date = datetime.date(dt.year, 1, 1) From 9c75336ea0e11d31adcf23b129f351d148b2569a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 13:02:09 +0530 Subject: [PATCH 28/79] fix: Set start of the week for date control based on configuration --- frappe/public/js/frappe/form/controls/date.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index 28e7f2a478..b03ea53239 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -62,6 +62,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat dateFormat: date_format, startDate: this.get_start_date(), keyboardNav: false, + firstDay: frappe.datetime.get_week_starts_on_index(), onSelect: () => { this.$input.trigger('change'); }, From 4a5d9a02e3b74b5c430f45514dc6fc5ccb3b7aa8 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 13:04:10 +0530 Subject: [PATCH 29/79] test: Update date utils test cases --- frappe/tests/test_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 67586719c8..5d2b8a918f 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -15,6 +15,8 @@ import io from mimetypes import guess_type from datetime import datetime, timedelta, date +from unittest.mock import patch + class TestFilters(unittest.TestCase): def test_simple_dict(self): self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open'}, {'status': 'Open'})) @@ -309,6 +311,14 @@ class TestDiffUtils(unittest.TestCase): class TestDateUtils(unittest.TestCase): def test_first_day_of_week(self): + # Monday as start of the week + with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-25"), + frappe.utils.getdate("2020-12-21")) + self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-20"), + frappe.utils.getdate("2020-12-14")) + + # Sunday as start of the week self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-25"), frappe.utils.getdate("2020-12-20")) self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-21"), From ca9b2a953d13a3c2b67a37c9aaba2da0d93fb8cc Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 13:25:19 +0530 Subject: [PATCH 30/79] fix: Add patch to set week_starts_on as "Monday" --- frappe/patches.txt | 1 + frappe/patches/v13_0/set_week_starts_on.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 frappe/patches/v13_0/set_week_starts_on.py diff --git a/frappe/patches.txt b/frappe/patches.txt index 27ba1a145d..3bd883aa6d 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -182,6 +182,7 @@ frappe.patches.v13_0.queryreport_columns execute:frappe.reload_doc('core', 'doctype', 'doctype') frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty +frappe.patches.v13_0.set_week_starts_on frappe.patches.v14_0.drop_data_import_legacy frappe.patches.v14_0.rename_cancelled_documents frappe.patches.v14_0.copy_mail_data #08.03.21 diff --git a/frappe/patches/v13_0/set_week_starts_on.py b/frappe/patches/v13_0/set_week_starts_on.py new file mode 100644 index 0000000000..8f90a6b2a8 --- /dev/null +++ b/frappe/patches/v13_0/set_week_starts_on.py @@ -0,0 +1,7 @@ +import frappe + +def execute(): + frappe.reload_doctype("System Settings") + # setting week_starts_on value as "Monday" to avoid breaking change + # because before the configuration was introduced, system used to consider "Monday" as start of the week + frappe.db.set_value("System Settings", "System Settings", "week_starts_on", "Monday") \ No newline at end of file From 5821a7bc9176ad8fb11d39bf89212d29066f327d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 13:40:30 +0530 Subject: [PATCH 31/79] test: Fix dashboard chart tests --- .../dashboard_chart/test_dashboard_chart.py | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 5562f2fc92..6e96c58464 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -8,6 +8,7 @@ from frappe.desk.doctype.dashboard_chart.dashboard_chart import get from datetime import datetime from dateutil.relativedelta import relativedelta +from unittest.mock import patch class TestDashboardChart(unittest.TestCase): def test_period_ending(self): @@ -15,8 +16,9 @@ class TestDashboardChart(unittest.TestCase): getdate('2019-04-10')) # week starts on monday - self.assertEqual(get_period_ending('2019-04-10', 'Weekly'), - getdate('2019-04-14')) + with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + self.assertEqual(get_period_ending('2019-04-10', 'Weekly'), + getdate('2019-04-14')) self.assertEqual(get_period_ending('2019-04-10', 'Monthly'), getdate('2019-04-30')) @@ -200,13 +202,14 @@ class TestDashboardChart(unittest.TestCase): timeseries = 1 )).insert() - result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1) + with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1) - self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0]) - self.assertEqual( - result.get('labels'), - ['30-12-18', '06-01-19', '13-01-19', '20-01-19'] - ) + self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0]) + self.assertEqual( + result.get('labels'), + ['30-12-18', '06-01-19', '13-01-19', '20-01-19'] + ) frappe.db.rollback() @@ -231,13 +234,13 @@ class TestDashboardChart(unittest.TestCase): timeseries = 1 )).insert() - result = get(chart_name='Test Average Dashboard Chart', refresh = 1) - - self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 150.0, 266.6666666666667, 0.0]) - self.assertEqual( - result.get('labels'), - ['30-12-18', '06-01-19', '13-01-19', '20-01-19'] - ) + with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + result = get(chart_name='Test Average Dashboard Chart', refresh = 1) + self.assertEqual( + result.get('labels'), + ['30-12-18', '06-01-19', '13-01-19', '20-01-19'] + ) + self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 150.0, 266.6666666666667, 0.0]) frappe.db.rollback() From ea2101d537cab6541f047f9fe1879f8303474a5a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 13:42:52 +0530 Subject: [PATCH 32/79] style: Fix formatting issues --- frappe/desk/doctype/system_console/system_console.js | 2 +- frappe/public/js/frappe/desk.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js index 06f04e8b32..ad0e9d638e 100644 --- a/frappe/desk/doctype/system_console/system_console.js +++ b/frappe/desk/doctype/system_console/system_console.js @@ -103,6 +103,6 @@ frappe.ui.form.on('System Console', { }, week_starts_on() { - frappe.app.setup_moment() + frappe.app.setup_moment(); } }); diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index f9a9931c4c..3668f0d3e7 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -627,13 +627,13 @@ frappe.Application = class Application { setup_moment() { moment.updateLocale('en', { - week : { - dow : frappe.datetime.get_week_starts_on_index(), + week: { + dow: frappe.datetime.get_week_starts_on_index(), } }); moment.locale("en"); moment.user_utc_offset = moment().utcOffset(); - if(frappe.boot.timezone_info) { + if (frappe.boot.timezone_info) { moment.tz.add(frappe.boot.timezone_info); } } From 5ac79925ef64d904bfd1df373e4d701e221948d9 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 4 Jan 2022 11:24:28 +0530 Subject: [PATCH 33/79] fix: postgres install that doesn't abort transactions --- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/utils/install.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index ad0c3e8e6f..cb62914fa0 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -1283,7 +1283,7 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): roles = [p.role for p in doc.get("permissions") or []] + default_roles for role in list(set(roles)): - if not frappe.db.exists("Role", role): + if frappe.db.table_exists("Role") and not frappe.db.exists("Role", role): r = frappe.get_doc(dict(doctype= "Role", role_name=role, desk_access=1)) r.flags.ignore_mandatory = r.flags.ignore_permissions = True r.insert() diff --git a/frappe/utils/install.py b/frappe/utils/install.py index bdc5fdfbc5..5ca8c4878a 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -5,11 +5,11 @@ import getpass from frappe.utils.password import update_password def before_install(): + frappe.reload_doc("core", "doctype", "doctype_state") frappe.reload_doc("core", "doctype", "docfield") frappe.reload_doc("core", "doctype", "docperm") frappe.reload_doc("core", "doctype", "doctype_action") frappe.reload_doc("core", "doctype", "doctype_link") - frappe.reload_doc("core", "doctype", "doctype_state") frappe.reload_doc("desk", "doctype", "form_tour_step") frappe.reload_doc("desk", "doctype", "form_tour") frappe.reload_doc("core", "doctype", "doctype") From 43364cf89b5a9755f234c08433f1146daa1a30c2 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 4 Jan 2022 11:47:50 +0530 Subject: [PATCH 34/79] fix!: use repeatable read isolation level RR isolation is default in MariaDB, for sake of consistency use same isolation level in postgres --- frappe/database/postgres/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index c163b72d14..e802b4f880 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -3,7 +3,7 @@ from typing import List, Tuple, Union import psycopg2 import psycopg2.extensions -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION import frappe @@ -69,7 +69,7 @@ class PostgresDatabase(Database): conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format( self.host, self.user, self.user, self.password, self.port )) - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) # TODO: Remove this + conn.set_isolation_level(ISOLATION_LEVEL_REPEATABLE_READ) return conn From 5fac8bb9653b0a5d80a431fee649c43aa9758b62 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 4 Jan 2022 12:36:23 +0530 Subject: [PATCH 35/79] test: don't hardcode test password --- frappe/tests/test_frappe_client.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/frappe/tests/test_frappe_client.py b/frappe/tests/test_frappe_client.py index 66e1160eea..e84163eb41 100644 --- a/frappe/tests/test_frappe_client.py +++ b/frappe/tests/test_frappe_client.py @@ -10,8 +10,9 @@ import requests import base64 class TestFrappeClient(unittest.TestCase): + PASSWORD = "admin" def test_insert_many(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Note", {"title": ("in", ('Sing','a','song','of','sixpence'))}) frappe.db.commit() @@ -30,7 +31,7 @@ class TestFrappeClient(unittest.TestCase): self.assertTrue(frappe.db.get_value('Note', {'title': 'sixpence'})) def test_create_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Note", {"title": "test_create"}) frappe.db.commit() @@ -39,13 +40,13 @@ class TestFrappeClient(unittest.TestCase): self.assertTrue(frappe.db.get_value('Note', {'title': 'test_create'})) def test_list_docs(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) doc_list = server.get_list("Note") self.assertTrue(len(doc_list)) def test_get_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Note", {"title": "get_this"}) frappe.db.commit() @@ -56,7 +57,7 @@ class TestFrappeClient(unittest.TestCase): self.assertTrue(doc) def test_get_value(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Note", {"title": "get_value"}) frappe.db.commit() @@ -74,14 +75,14 @@ class TestFrappeClient(unittest.TestCase): self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"}) def test_get_single(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) server.set_value('Website Settings', 'Website Settings', 'title_prefix', 'test-prefix') self.assertEqual(server.get_value('Website Settings', 'title_prefix', 'Website Settings').get('title_prefix'), 'test-prefix') self.assertEqual(server.get_value('Website Settings', 'title_prefix').get('title_prefix'), 'test-prefix') frappe.db.set_value('Website Settings', None, 'title_prefix', '') def test_update_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Note", {"title": ("in", ("Sing", "sing"))}) frappe.db.commit() @@ -93,7 +94,7 @@ class TestFrappeClient(unittest.TestCase): self.assertTrue(doc["title"] == changed_title) def test_update_child_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Contact", {"first_name": "George", "last_name": "Steevens"}) frappe.db.delete("Contact", {"first_name": "William", "last_name": "Shakespeare"}) frappe.db.delete("Communication", {"reference_doctype": "Event"}) @@ -130,7 +131,7 @@ class TestFrappeClient(unittest.TestCase): self.assertTrue(frappe.db.exists("Communication Link", {"link_name": "William Shakespeare"})) def test_delete_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Note", {"title": "delete"}) frappe.db.commit() From 08464553f309370aa5d0dd138291a58bfa5f4783 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 4 Jan 2022 13:00:57 +0530 Subject: [PATCH 36/79] fix: define check_transaction_status for postgres To implement repeatable reads it has to respect transactions. --- frappe/database/database.py | 9 ++++++--- frappe/database/postgres/database.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 856c85416f..30689d44aa 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -267,9 +267,7 @@ class Database(object): """Raises exception if more than 20,000 `INSERT`, `UPDATE` queries are executed in one transaction. This is to ensure that writes are always flushed otherwise this could cause the system to hang.""" - if self.transaction_writes and \ - query and query.strip().split()[0].lower() in ['start', 'alter', 'drop', 'create', "begin", "truncate"]: - raise Exception('This statement can cause implicit commit') + self.check_implicit_commit(query) if query and query.strip().lower() in ('commit', 'rollback'): self.transaction_writes = 0 @@ -282,6 +280,11 @@ class Database(object): else: frappe.throw(_("Too many writes in one request. Please send smaller requests"), frappe.ValidationError) + def check_implicit_commit(self, query): + if self.transaction_writes and \ + query and query.strip().split()[0].lower() in ['start', 'alter', 'drop', 'create', "begin", "truncate"]: + raise Exception('This statement can cause implicit commit') + def fetch_as_dict(self, formatted=0, as_utf8=0): """Internal. Converts results to dict.""" result = self._cursor.fetchall() diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index e802b4f880..9fd033768d 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -259,8 +259,8 @@ class PostgresDatabase(Database): key=key ) - def check_transaction_status(self, query): - pass + def check_implicit_commit(self, query): + pass # postgres can run DDL in transactions without implicit commits def has_index(self, table_name, index_name): return self.sql("""SELECT 1 FROM pg_indexes WHERE tablename='{table_name}' From da9d07474af65c1fd43276d019076e51433762dc Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 4 Jan 2022 13:19:56 +0530 Subject: [PATCH 37/79] test: rollback tests that are aborting transactions --- frappe/core/doctype/doctype/test_doctype.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 4362a52c34..12c227464d 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -15,6 +15,10 @@ from frappe.core.doctype.doctype.doctype import (UniqueFieldnameError, # test_records = frappe.get_test_records('DocType') class TestDocType(unittest.TestCase): + + def tearDown(self): + frappe.db.rollback() + def test_validate_name(self): self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert) self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert) @@ -42,6 +46,7 @@ class TestDocType(unittest.TestCase): doc1.insert() self.assertRaises(frappe.UniqueValidationError, doc2.insert) + frappe.db.rollback() dt.fields[0].unique = 0 dt.save() From 6b77ed68e9b74b85cf5bbee1c05630a8e36d2ebb Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 14:07:17 +0530 Subject: [PATCH 38/79] fix: Revert unnecessary code --- frappe/desk/doctype/system_console/system_console.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js index ad0e9d638e..fc83069fd2 100644 --- a/frappe/desk/doctype/system_console/system_console.js +++ b/frappe/desk/doctype/system_console/system_console.js @@ -101,8 +101,4 @@ frappe.ui.form.on('System Console', { ${rows}`); }); }, - - week_starts_on() { - frappe.app.setup_moment(); - } }); From 53b2c472a6f4576ef7fbeea478d54b1ec54ba7ae Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 14:14:06 +0530 Subject: [PATCH 39/79] fix: Re-setup moment object whenever `week_starts_on` is changed --- frappe/core/doctype/system_settings/system_settings.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index 4eeab0274b..cb1686df39 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -10,6 +10,10 @@ frappe.ui.form.on("System Settings", { frm.set_value(key, val); frappe.sys_defaults[key] = val; }); + if (frm.re_setup_moment) { + frappe.app.setup_moment(); + delete frm.re_setup_moment; + } } }); }, @@ -38,5 +42,8 @@ frappe.ui.form.on("System Settings", { // Clear cache after saving to refresh the values of boot. frappe.ui.toolbar.clear_cache(); } - } + }, + week_starts_on(frm) { + frm.re_setup_moment = true; + }, }); From 4b9d8c025850a72e779e066be9858ec15da372b2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 4 Jan 2022 15:31:08 +0530 Subject: [PATCH 40/79] fix: Remove extra Document.validate_owner validation --- frappe/model/document.py | 6 ------ frappe/tests/test_document.py | 19 ------------------- 2 files changed, 25 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index f199c96acd..e25469c68a 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -504,7 +504,6 @@ class Document(BaseDocument): self._sanitize_content() self._save_passwords() self.validate_workflow() - self.validate_owner() children = self.get_all_children() for d in children: @@ -547,11 +546,6 @@ class Document(BaseDocument): if not self._action == 'save': set_workflow_state_on_action(self, workflow, self._action) - def validate_owner(self): - """Validate if the owner of the Document has changed""" - if not self.is_new() and self.has_value_changed('owner'): - frappe.throw(_('Document owner cannot be changed')) - def validate_set_only_once(self): """Validate that fields are not changed if not in insert""" set_only_once_fields = self.meta.get_set_only_once_fields() diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 46638f5bf2..29cec8b230 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -252,22 +252,3 @@ class TestDocument(unittest.TestCase): 'currency': 100000 }) self.assertEquals(d.get_formatted('currency', currency='INR', format="#,###.##"), '₹ 100,000.00') - - def test_owner_changed(self): - frappe.delete_doc_if_exists("User", "hello@example.com") - frappe.set_user("Administrator") - - d = frappe.get_doc({ - "doctype": "User", - "email": "hello@example.com", - "first_name": "John" - }) - d.insert() - self.assertEqual(frappe.db.get_value("User", d.owner), d.owner) - - d.set("owner", "johndoe@gmail.com") - self.assertRaises(frappe.ValidationError, d.save) - - d.reload() - d.save() - self.assertEqual(frappe.db.get_value("User", d.owner), d.owner) From 9ed95a2d75b2d36be0fa6bc53ed072ad49d2b0d8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 4 Jan 2022 15:32:26 +0530 Subject: [PATCH 41/79] chore: Update caniuse-lite package --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index d5bc1d669f..ba58f6b719 100644 --- a/yarn.lock +++ b/yarn.lock @@ -554,9 +554,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001196, caniuse-lite@^1.0.30001219: - version "1.0.30001272" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001272.tgz" - integrity sha512-DV1j9Oot5dydyH1v28g25KoVm7l8MTxazwuiH3utWiAS6iL/9Nh//TGwqFEeqqN8nnWYQ8HHhUq+o4QPt9kvYw== + version "1.0.30001296" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz" + integrity sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q== caseless@~0.12.0: version "0.12.0" From 22dd06101b25ace52f562624d8653735cf9df1dd Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 4 Jan 2022 17:19:10 +0530 Subject: [PATCH 42/79] fix(blog): Load more with category filter When you click "Load more" on a blog category page, it will now fetch blog posts for that category --- .../blog_post/templates/blog_post_list.html | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/frappe/website/doctype/blog_post/templates/blog_post_list.html b/frappe/website/doctype/blog_post/templates/blog_post_list.html index 2b3d5e250c..1aa30316fe 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post_list.html +++ b/frappe/website/doctype/blog_post/templates/blog_post_list.html @@ -52,5 +52,39 @@ {% endblock %} {% block script %} - + {% endblock %} From dc2a99b8d671e21f2a50765c3ef950a73fec883d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 4 Jan 2022 18:57:49 +0530 Subject: [PATCH 43/79] refactor(minor): Use pluck instead of re-iteration --- frappe/desk/doctype/todo/todo.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index eabb28a6f3..e689faafbe 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -69,15 +69,13 @@ class ToDo(Document): return try: - assignments = [d[0] for d in frappe.get_all("ToDo", - filters={ - "reference_type": self.reference_type, - "reference_name": self.reference_name, - "status": ("!=", "Cancelled") - }, - fields=["allocated_to"], as_list=True)] - + assignments = frappe.get_all("ToDo", filters={ + "reference_type": self.reference_type, + "reference_name": self.reference_name, + "status": ("!=", "Cancelled") + }, pluck="allocated_to") assignments.reverse() + frappe.db.set_value(self.reference_type, self.reference_name, "_assign", json.dumps(assignments), update_modified=False) From 1b57717058eb89f1f0b9b17ef462b3720dd43ce8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 4 Jan 2022 18:58:17 +0530 Subject: [PATCH 44/79] fix: Return key name as owner for consistency --- frappe/desk/form/assign_to.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/desk/form/assign_to.py b/frappe/desk/form/assign_to.py index 7ea87b8d15..049d33c1ec 100644 --- a/frappe/desk/form/assign_to.py +++ b/frappe/desk/form/assign_to.py @@ -19,11 +19,11 @@ def get(args=None): if not args: args = frappe.local.form_dict - return frappe.get_all('ToDo', fields=['allocated_to', 'name'], filters=dict( - reference_type = args.get('doctype'), - reference_name = args.get('name'), - status = ('!=', 'Cancelled') - ), limit=5) + return frappe.get_all("ToDo", fields=["allocated_to as owner", "name"], filters={ + "reference_type": args.get("doctype"), + "reference_name": args.get("name"), + "status": ("!=", "Cancelled") + }, limit=5) @frappe.whitelist() def add(args=None): From 6e4aa52e2ecda5f8aaa61dad8461c1f519aa4053 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 4 Jan 2022 19:20:03 +0530 Subject: [PATCH 45/79] fix: Show assignments correctly for Forms --- frappe/desk/form/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 89e6598859..0e644c3cf5 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -253,7 +253,7 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields= def get_assignments(dt, dn): cl = frappe.get_all("ToDo", - fields=['name', 'owner', 'description', 'status'], + fields=['name', 'allocated_to as owner', 'description', 'status'], filters={ 'reference_type': dt, 'reference_name': dn, From f6d069730981f84c08dfe17279a8d67b83740f69 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Jan 2022 19:54:41 +0530 Subject: [PATCH 46/79] test: Fix test_assign --- frappe/tests/test_assign.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_assign.py b/frappe/tests/test_assign.py index 971f9ce071..05bf7e2fb3 100644 --- a/frappe/tests/test_assign.py +++ b/frappe/tests/test_assign.py @@ -13,7 +13,7 @@ class TestAssign(unittest.TestCase): added = assign(todo, "test@example.com") - self.assertTrue("test@example.com" in [d.allocated_to for d in added]) + self.assertTrue("test@example.com" in [d.owner for d in added]) removed = frappe.desk.form.assign_to.remove(todo.doctype, todo.name, "test@example.com") From c99e576e1b7e12154e4f1b174a93ea0ecf80899c Mon Sep 17 00:00:00 2001 From: Christoph Kappel Date: Mon, 3 Jan 2022 22:51:29 +0100 Subject: [PATCH 47/79] fix: offer all (also modern) supported tls versions (PROTOCOL_TLS_CLIENT [1]) to LDAP endpoints instead of only (deprecated) PROTOCOL_TLSv1 [2] Background: Currently, when connecting to a ldap backend, ssl.PROTOCOL_TLSv1 [2] is offered as only option to the backend. This leads to following issues: - LDAP Backends that do not support TLSv1.0 (because of security reasons [3]) cannot be used in ERPNext - erpnext can ONLY connect to LDAP Backends offering the insecure [3] TLSv1.0 protocol (see ldap_settings.py ln: 61, 63) With this change to ssl.PROTOCOL_TLS_CLIENT we allow erpnext customers to configure LDAP Backends that also support more modern/secure (TLSv1.2 and up) transport while still ensure backwards compatibility and allowing TLSv1.0, since ssl.PROTOCOL_TLS "Auto-negotiates the highest protocol version that both the client and server support" [1] [1]: https://docs.python.org/3/library/ssl.html#ssl.PROTOCOL_TLS_CLIENT [2]: https://docs.python.org/3/library/ssl.html#ssl.PROTOCOL_TLSv1 [3]: https://tools.ietf.org/id/draft-ietf-tls-oldversions-deprecate-02.html --- frappe/integrations/doctype/ldap_settings/ldap_settings.py | 4 ++-- .../integrations/doctype/ldap_settings/test_ldap_settings.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index 1c5abb454c..7c9c64ba3c 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -58,9 +58,9 @@ class LDAPSettings(Document): import ssl if self.require_trusted_certificate == 'Yes': - tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLSv1) + tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLS_CLIENT) else: - tls_configuration = ldap3.Tls(validate=ssl.CERT_NONE, version=ssl.PROTOCOL_TLSv1) + tls_configuration = ldap3.Tls(validate=ssl.CERT_NONE, version=ssl.PROTOCOL_TLS_CLIENT) if self.local_private_key_file: tls_configuration.private_key_file = self.local_private_key_file diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py index 7b0638876b..41997fb4c7 100644 --- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py @@ -296,7 +296,7 @@ class LDAP_TestCase(): if local_doc['require_trusted_certificate'] == 'Yes': tls_validate = ssl.CERT_REQUIRED - tls_version = ssl.PROTOCOL_TLSv1 + tls_version = ssl.PROTOCOL_TLS_CLIENT tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version) self.assertTrue(kwargs['auto_bind'] == ldap3.AUTO_BIND_TLS_BEFORE_BIND, @@ -304,7 +304,7 @@ class LDAP_TestCase(): else: tls_validate = ssl.CERT_NONE - tls_version = ssl.PROTOCOL_TLSv1 + tls_version = ssl.PROTOCOL_TLS_CLIENT tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version) self.assertTrue(kwargs['auto_bind'], From 80d456ef7e9f0b96ac5081a11545773c2bc398c2 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 5 Jan 2022 11:53:03 +0530 Subject: [PATCH 48/79] fix: avoid cached results for `table_exists` during install --- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/database/database.py | 4 ++-- frappe/database/postgres/database.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index cb62914fa0..3754288145 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -1283,7 +1283,7 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): roles = [p.role for p in doc.get("permissions") or []] + default_roles for role in list(set(roles)): - if frappe.db.table_exists("Role") and not frappe.db.exists("Role", role): + if frappe.db.table_exists("Role", cached=False) and not frappe.db.exists("Role", role): r = frappe.get_doc(dict(doctype= "Role", role_name=role, desk_access=1)) r.flags.ignore_mandatory = r.flags.ignore_permissions = True r.insert() diff --git a/frappe/database/database.py b/frappe/database/database.py index 30689d44aa..993e2eb51c 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -838,9 +838,9 @@ class Database(object): 'parent': dt }) - def table_exists(self, doctype): + def table_exists(self, doctype, cached=True): """Returns True if table for given doctype exists.""" - return ("tab" + doctype) in self.get_tables() + return ("tab" + doctype) in self.get_tables(cached=cached) def has_table(self, doctype): return self.table_exists(doctype) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 9fd033768d..0ce6fbb265 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -103,7 +103,7 @@ class PostgresDatabase(Database): return super(PostgresDatabase, self).sql(*args, **kwargs) - def get_tables(self): + def get_tables(self, cached=True): return [d[0] for d in self.sql("""select table_name from information_schema.tables where table_catalog='{0}' From ae5644c3e188522c5db2e0e73762f08f46961738 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Wed, 5 Jan 2022 14:23:44 +0530 Subject: [PATCH 49/79] fix: set `first_response_time` only if communication is sent --- frappe/core/doctype/communication/communication.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 3a78a6a599..96c8f271d9 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -488,10 +488,12 @@ def update_parent_document_on_communication(doc): def update_first_response_time(parent, communication): if parent.meta.has_field("first_response_time") and not parent.get("first_response_time"): if is_system_user(communication.sender): - first_responded_on = communication.creation - if parent.meta.has_field("first_responded_on") and communication.sent_or_received == "Sent": - parent.db_set("first_responded_on", first_responded_on) - parent.db_set("first_response_time", round(time_diff_in_seconds(first_responded_on, parent.creation), 2)) + if communication.sent_or_received == "Sent": + first_responded_on = communication.creation + if parent.meta.has_field("first_responded_on"): + parent.db_set("first_responded_on", first_responded_on) + first_response_time = round(time_diff_in_seconds(first_responded_on, parent.creation), 2) + parent.db_set("first_response_time", first_response_time) def set_avg_response_time(parent, communication): if parent.meta.has_field("avg_response_time") and communication.sent_or_received == "Sent": From 5d5ad78789738e6c9ddd3a368f44badc9022ce5a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 5 Jan 2022 14:54:46 +0530 Subject: [PATCH 50/79] refactor: Rename `week_starts_on` to `first_day_of_the_week` For consistency --- frappe/core/doctype/system_settings/system_settings.js | 2 +- frappe/core/doctype/system_settings/system_settings.json | 6 +++--- frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py | 6 +++--- frappe/patches.txt | 2 +- .../{set_week_starts_on.py => set_first_day_of_the_week.py} | 4 ++-- frappe/public/js/frappe/desk.js | 2 +- frappe/public/js/frappe/form/controls/date.js | 2 +- frappe/public/js/frappe/utils/datetime.js | 6 +++--- frappe/tests/test_utils.py | 2 +- frappe/utils/data.py | 6 +++--- 10 files changed, 19 insertions(+), 19 deletions(-) rename frappe/patches/v13_0/{set_week_starts_on.py => set_first_day_of_the_week.py} (51%) diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index cb1686df39..5128ae24cb 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -43,7 +43,7 @@ frappe.ui.form.on("System Settings", { frappe.ui.toolbar.clear_cache(); } }, - week_starts_on(frm) { + first_day_of_the_week(frm) { frm.re_setup_moment = true; }, }); diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 0af8f2e4be..61410fb1a8 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -21,7 +21,7 @@ "column_break_7", "float_precision", "currency_precision", - "week_starts_on", + "first_day_of_the_week", "sec_backup_limit", "backup_limit", "encrypt_backup", @@ -481,9 +481,9 @@ }, { "default": "Sunday", - "fieldname": "week_starts_on", + "fieldname": "first_day_of_the_week", "fieldtype": "Select", - "label": "Week Starts On", + "label": "First Day of the Week", "options": "Sunday\nMonday\nTuesday\nWednesday\nThursday\nFriday\nSaturday" } ], diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 6e96c58464..5c986b5b7c 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -16,7 +16,7 @@ class TestDashboardChart(unittest.TestCase): getdate('2019-04-10')) # week starts on monday - with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): self.assertEqual(get_period_ending('2019-04-10', 'Weekly'), getdate('2019-04-14')) @@ -202,7 +202,7 @@ class TestDashboardChart(unittest.TestCase): timeseries = 1 )).insert() - with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1) self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0]) @@ -234,7 +234,7 @@ class TestDashboardChart(unittest.TestCase): timeseries = 1 )).insert() - with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): result = get(chart_name='Test Average Dashboard Chart', refresh = 1) self.assertEqual( result.get('labels'), diff --git a/frappe/patches.txt b/frappe/patches.txt index 2e94185179..af7e4d6e3f 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -182,7 +182,7 @@ frappe.patches.v13_0.queryreport_columns execute:frappe.reload_doc('core', 'doctype', 'doctype') frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty -frappe.patches.v13_0.set_week_starts_on +frappe.patches.v13_0.set_first_day_of_the_week frappe.patches.v14_0.drop_data_import_legacy frappe.patches.v14_0.rename_cancelled_documents frappe.patches.v14_0.copy_mail_data #08.03.21 diff --git a/frappe/patches/v13_0/set_week_starts_on.py b/frappe/patches/v13_0/set_first_day_of_the_week.py similarity index 51% rename from frappe/patches/v13_0/set_week_starts_on.py rename to frappe/patches/v13_0/set_first_day_of_the_week.py index 8f90a6b2a8..cfb694bbf1 100644 --- a/frappe/patches/v13_0/set_week_starts_on.py +++ b/frappe/patches/v13_0/set_first_day_of_the_week.py @@ -2,6 +2,6 @@ import frappe def execute(): frappe.reload_doctype("System Settings") - # setting week_starts_on value as "Monday" to avoid breaking change + # setting first_day_of_the_week value as "Monday" to avoid breaking change # because before the configuration was introduced, system used to consider "Monday" as start of the week - frappe.db.set_value("System Settings", "System Settings", "week_starts_on", "Monday") \ No newline at end of file + frappe.db.set_value("System Settings", "System Settings", "first_day_of_the_week", "Monday") \ No newline at end of file diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 3668f0d3e7..202cee645a 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -628,7 +628,7 @@ frappe.Application = class Application { setup_moment() { moment.updateLocale('en', { week: { - dow: frappe.datetime.get_week_starts_on_index(), + dow: frappe.datetime.get_first_day_of_the_week_index(), } }); moment.locale("en"); diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index b03ea53239..78eb3832cc 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -62,7 +62,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat dateFormat: date_format, startDate: this.get_start_date(), keyboardNav: false, - firstDay: frappe.datetime.get_week_starts_on_index(), + firstDay: frappe.datetime.get_first_day_of_the_week_index(), onSelect: () => { this.$input.trigger('change'); }, diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 09e44d917d..196bdf68a3 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -254,9 +254,9 @@ $.extend(frappe.datetime, { ], true).isValid(); }, - get_week_starts_on_index() { - const week_starts_on = frappe.sys_defaults.week_starts_on || "Sunday"; - return moment.weekdays().indexOf(week_starts_on); + get_first_day_of_the_week_index() { + const first_day_of_the_week = frappe.sys_defaults.first_day_of_the_week || "Sunday"; + return moment.weekdays().indexOf(first_day_of_the_week); } }); diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 5d2b8a918f..5c1541e0de 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -312,7 +312,7 @@ class TestDiffUtils(unittest.TestCase): class TestDateUtils(unittest.TestCase): def test_first_day_of_week(self): # Monday as start of the week - with patch.object(frappe.utils.data, "get_week_starts_on", return_value="Monday"): + with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-25"), frappe.utils.getdate("2020-12-21")) self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-20"), diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 9f9391b1fc..545d49054a 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -26,11 +26,11 @@ class Weekday(Enum): Friday = 5 Saturday = 6 -def get_week_starts_on(): - return frappe.get_system_settings('week_starts_on') or "Sunday" +def get_first_day_of_the_week(): + return frappe.get_system_settings('first_day_of_the_week') or "Sunday" def get_start_of_week_index(): - return Weekday[get_week_starts_on()].value + return Weekday[get_first_day_of_the_week()].value def is_invalid_date_string(date_string): # dateutil parser does not agree with dates like "0001-01-01" or "0000-00-00" From d73428df919dc98e7ee13af92af5fd8a66da66d4 Mon Sep 17 00:00:00 2001 From: Navdeep Date: Mon, 26 Apr 2021 11:09:22 -0400 Subject: [PATCH 51/79] feat(fixes): for frappeclient post_api(POST) request being redirected to GET request on calling server --- frappe/frappeclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py index ab58979203..cf6afde4a6 100644 --- a/frappe/frappeclient.py +++ b/frappe/frappeclient.py @@ -296,7 +296,7 @@ class FrappeClient(object): def post_api(self, method, params=None): if params is None: params = {} - res = self.session.post(self.url + "/api/method/" + method + "/", + res = self.session.post(self.url + "/api/method/" + method, params=params, verify=self.verify, headers=self.headers) return self.post_process(res) From 984420363a9aa8d60f95b197e0750e1766347ddf Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 5 Jan 2022 20:37:33 +0530 Subject: [PATCH 52/79] fix: Use params instead of data for client login --- frappe/frappeclient.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py index cf6afde4a6..59db38584c 100644 --- a/frappe/frappeclient.py +++ b/frappe/frappeclient.py @@ -1,11 +1,13 @@ -import requests -import json -import frappe -import base64 - ''' FrappeClient is a library that helps you connect with other frappe systems ''' +import base64 +import json + +import requests + +import frappe + class AuthError(Exception): pass @@ -46,7 +48,7 @@ class FrappeClient(object): def _login(self, username, password): '''Login/start a sesion. Called internally on init''' - r = self.session.post(self.url, data={ + r = self.session.post(self.url, params={ 'cmd': 'login', 'usr': username, 'pwd': password @@ -289,14 +291,14 @@ class FrappeClient(object): def get_api(self, method, params=None): if params is None: params = {} - res = self.session.get(self.url + "/api/method/" + method + "/", + res = self.session.get(f"{self.url}/api/method/{method}", params=params, verify=self.verify, headers=self.headers) return self.post_process(res) def post_api(self, method, params=None): if params is None: params = {} - res = self.session.post(self.url + "/api/method/" + method, + res = self.session.post(f"{self.url}/api/method/{method}", params=params, verify=self.verify, headers=self.headers) return self.post_process(res) From bf28199a6dd14c63ac7feb0ec1ace67eeff57408 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Wed, 5 Jan 2022 18:14:54 +0100 Subject: [PATCH 53/79] fix: awesomeplete line wrapping --- frappe/public/scss/common/awesomeplete.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/scss/common/awesomeplete.scss b/frappe/public/scss/common/awesomeplete.scss index b9e8035d68..17f33d7e82 100644 --- a/frappe/public/scss/common/awesomeplete.scss +++ b/frappe/public/scss/common/awesomeplete.scss @@ -39,6 +39,7 @@ padding: var(--padding-sm); color: var(--text-color); border-radius: var(--border-radius); + white-space: unset; @extend .ellipsis; &:not(:last-child) { margin-bottom: var(--margin-xs); From 54fbd0b5e6239398e0833557e718ff5bf96d578d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 6 Jan 2022 09:05:08 +0530 Subject: [PATCH 54/79] test: UI test cases for "First Day of the Week" --- cypress/integration/first_day_of_the_week.js | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 cypress/integration/first_day_of_the_week.js diff --git a/cypress/integration/first_day_of_the_week.js b/cypress/integration/first_day_of_the_week.js new file mode 100644 index 0000000000..7099d3fddc --- /dev/null +++ b/cypress/integration/first_day_of_the_week.js @@ -0,0 +1,48 @@ +context("First Day of the Week", () => { + before(() => { + cy.login(); + }); + + beforeEach(() => { + cy.visit('/app/system-settings'); + cy.findByText('Date and Number Format').click(); + cy.fill_field('first_day_of_the_week', 'Sunday', 'Select'); + cy.findByRole('button', {name: 'Save'}).click(); + cy.findByText('Date and Number Format').click(); + }); + + it("Date control starts with same day as selected in System Settings", () => { + cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings") + cy.fill_field('first_day_of_the_week', 'Tuesday', 'Select'); + cy.findByRole('button', {name: 'Save'}).click(); + cy.wait("@load_settings"); + cy.dialog({ + title: 'Date', + fields: [ + { + label: 'Date', + fieldname: 'date', + fieldtype: 'Date' + } + ] + }); + cy.get_field('date').click(); + cy.get('.datepicker--day-name').eq(0).should('have.text', 'Tu'); + }); + + it("Calendar view starts with same day as selected in System Settings", () => { + cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings") + cy.fill_field('first_day_of_the_week', 'Monday', 'Select'); + cy.findByRole('button', {name: 'Save'}).click(); + cy.wait("@load_settings"); + cy.visit("app/todo/view/calendar/default"); + cy.get('.fc-day-header > span').eq(0).should('have.text', 'Mon'); + }); + + after(() => { + cy.visit('/app/system-settings'); + cy.findByText('Date and Number Format').click(); + cy.fill_field('first_day_of_the_week', 'Sunday', 'Select'); + cy.findByRole('button', {name: 'Save'}).click(); + }); +}) \ No newline at end of file From ed2e37eaf7c61e0687d9d817505b10b2654c8f4a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 6 Jan 2022 09:18:42 +0530 Subject: [PATCH 55/79] style: Fix sider issues --- .eslintrc | 1 + cypress/integration/first_day_of_the_week.js | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.eslintrc b/.eslintrc index cc7f555669..937f11586c 100644 --- a/.eslintrc +++ b/.eslintrc @@ -148,6 +148,7 @@ "context": true, "before": true, "beforeEach": true, + "after": true, "qz": true, "localforage": true, "extend_cscript": true diff --git a/cypress/integration/first_day_of_the_week.js b/cypress/integration/first_day_of_the_week.js index 7099d3fddc..2626905f5b 100644 --- a/cypress/integration/first_day_of_the_week.js +++ b/cypress/integration/first_day_of_the_week.js @@ -12,7 +12,7 @@ context("First Day of the Week", () => { }); it("Date control starts with same day as selected in System Settings", () => { - cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings") + cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings"); cy.fill_field('first_day_of_the_week', 'Tuesday', 'Select'); cy.findByRole('button', {name: 'Save'}).click(); cy.wait("@load_settings"); @@ -31,7 +31,7 @@ context("First Day of the Week", () => { }); it("Calendar view starts with same day as selected in System Settings", () => { - cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings") + cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings"); cy.fill_field('first_day_of_the_week', 'Monday', 'Select'); cy.findByRole('button', {name: 'Save'}).click(); cy.wait("@load_settings"); @@ -45,4 +45,4 @@ context("First Day of the Week", () => { cy.fill_field('first_day_of_the_week', 'Sunday', 'Select'); cy.findByRole('button', {name: 'Save'}).click(); }); -}) \ No newline at end of file +}); \ No newline at end of file From 30591720f868e924973c2fd458448752d8a5295c Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 6 Jan 2022 10:18:12 +0530 Subject: [PATCH 56/79] test: Fix flaky grid flaky test Simplified test case as bit --- cypress/integration/grid_keyboard_shortcut.js | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/cypress/integration/grid_keyboard_shortcut.js b/cypress/integration/grid_keyboard_shortcut.js index dee056e03e..189ed0812c 100644 --- a/cypress/integration/grid_keyboard_shortcut.js +++ b/cypress/integration/grid_keyboard_shortcut.js @@ -1,48 +1,39 @@ context('Grid Keyboard Shortcut', () => { let total_count = 0; - beforeEach(() => { - cy.login(); - cy.visit('/app/doctype/User'); - }); before(() => { cy.login(); - cy.visit('/app/doctype/User'); - return cy.window().its('frappe').then(frappe => { - frappe.db.count('DocField', { - filters: { - 'parent': 'User', 'parentfield': 'fields', 'parenttype': 'DocType' - } - }).then((r) => { - total_count = r; - }); - }); + }) + beforeEach(() => { + cy.reload(); + cy.visit('/app/contact/new-contact-1'); + cy.get('.frappe-control[data-fieldname="email_ids"]').find(".grid-add-row").click() }); it('Insert new row at the end', () => { cy.add_new_row_in_grid('{ctrl}{shift}{downarrow}', (cy, total_count) => { - cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', `${total_count+1}`); + cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', `${total_count+1}`); }, total_count); }); it('Insert new row at the top', () => { cy.add_new_row_in_grid('{ctrl}{shift}{uparrow}', (cy) => { - cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '1'); + cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2'); }); }); it('Insert new row below', () => { cy.add_new_row_in_grid('{ctrl}{downarrow}', (cy) => { - cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '2'); + cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '1'); }); }); it('Insert new row above', () => { cy.add_new_row_in_grid('{ctrl}{uparrow}', (cy) => { - cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '1'); + cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2'); }); }); }); Cypress.Commands.add('add_new_row_in_grid', (shortcut_keys, callbackFn, total_count) => { - cy.get('.frappe-control[data-fieldname="fields"]').as('table'); - cy.get('@table').find('.grid-body .col-xs-2').first().click(); - cy.get('@table').find('.grid-body .col-xs-2') + cy.get('.frappe-control[data-fieldname="email_ids"]').as('table'); + cy.get('@table').find('.grid-body [data-fieldname="email_id"]').first().click(); + cy.get('@table').find('.grid-body [data-fieldname="email_id"]') .first().type(shortcut_keys); callbackFn(cy, total_count); From 454a08aa183d81cbaed1c3ff789a5943c92ddd82 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 6 Jan 2022 10:43:04 +0530 Subject: [PATCH 57/79] test: Remove unnecessary code --- cypress/integration/first_day_of_the_week.js | 3 --- cypress/integration/grid_keyboard_shortcut.js | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cypress/integration/first_day_of_the_week.js b/cypress/integration/first_day_of_the_week.js index 2626905f5b..1e65b78990 100644 --- a/cypress/integration/first_day_of_the_week.js +++ b/cypress/integration/first_day_of_the_week.js @@ -6,9 +6,6 @@ context("First Day of the Week", () => { beforeEach(() => { cy.visit('/app/system-settings'); cy.findByText('Date and Number Format').click(); - cy.fill_field('first_day_of_the_week', 'Sunday', 'Select'); - cy.findByRole('button', {name: 'Save'}).click(); - cy.findByText('Date and Number Format').click(); }); it("Date control starts with same day as selected in System Settings", () => { diff --git a/cypress/integration/grid_keyboard_shortcut.js b/cypress/integration/grid_keyboard_shortcut.js index 189ed0812c..9cf39165ad 100644 --- a/cypress/integration/grid_keyboard_shortcut.js +++ b/cypress/integration/grid_keyboard_shortcut.js @@ -2,11 +2,11 @@ context('Grid Keyboard Shortcut', () => { let total_count = 0; before(() => { cy.login(); - }) + }); beforeEach(() => { cy.reload(); cy.visit('/app/contact/new-contact-1'); - cy.get('.frappe-control[data-fieldname="email_ids"]').find(".grid-add-row").click() + cy.get('.frappe-control[data-fieldname="email_ids"]').find(".grid-add-row").click(); }); it('Insert new row at the end', () => { cy.add_new_row_in_grid('{ctrl}{shift}{downarrow}', (cy, total_count) => { From 997c0e25cbb6df999ef51251d13ec790bfaf17ec Mon Sep 17 00:00:00 2001 From: harshpwctech <84438948+harshpwctech@users.noreply.github.com> Date: Thu, 6 Jan 2022 11:24:58 +0530 Subject: [PATCH 58/79] Updating the get_payment_gateway_url method (#15423) This change is proposed as on_payment_authorized method defined in any doctype for which the payment is made isn't getting triggered. https://github.com/frappe/frappe/pull/12756 --- frappe/website/doctype/web_form/web_form.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index eb7ee90826..cef1a8d8a3 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -216,8 +216,8 @@ def get_context(context): "amount": amount, "title": title, "description": title, - "reference_doctype": "Web Form", - "reference_docname": self.name, + "reference_doctype": doc.doctype, + "reference_docname": doc.name, "payer_email": frappe.session.user, "payer_name": frappe.utils.get_fullname(frappe.session.user), "order_id": doc.name, From c8c77437fa70080a14ac22bcfce92f696ac041bb Mon Sep 17 00:00:00 2001 From: hrwx Date: Thu, 6 Jan 2022 12:02:24 +0530 Subject: [PATCH 59/79] feat: add translations to datatable --- frappe/public/js/desk.bundle.js | 1 + .../js/frappe/data_import/import_preview.js | 2 ++ frappe/public/js/frappe/utils/datatable.js | 22 +++++++++++++++++++ .../js/frappe/views/reports/query_report.js | 4 +++- .../js/frappe/views/reports/report_view.js | 2 ++ 5 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 frappe/public/js/frappe/utils/datatable.js diff --git a/frappe/public/js/desk.bundle.js b/frappe/public/js/desk.bundle.js index 99b644a5c7..cac02c7a68 100644 --- a/frappe/public/js/desk.bundle.js +++ b/frappe/public/js/desk.bundle.js @@ -63,6 +63,7 @@ import "./frappe/utils/address_and_contact.js"; import "./frappe/utils/preview_email.js"; import "./frappe/utils/file_manager.js"; import "./frappe/utils/diffview"; +import "./frappe/utils/datatable.js"; import "./frappe/upload.js"; import "./frappe/ui/tree.js"; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 75bfb90bde..2264042539 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -142,6 +142,8 @@ frappe.data_import.ImportPreview = class ImportPreview { columns: this.columns, layout: this.columns.length < 10 ? 'fluid' : 'fixed', cellHeight: 35, + language: frappe.boot.lang, + translations: frappe.utils.datatable.get_translations(), serialNoColumn: false, checkboxColumn: false, noDataMessage: __('No Data'), diff --git a/frappe/public/js/frappe/utils/datatable.js b/frappe/public/js/frappe/utils/datatable.js new file mode 100644 index 0000000000..e07b86df94 --- /dev/null +++ b/frappe/public/js/frappe/utils/datatable.js @@ -0,0 +1,22 @@ +frappe.provide("frappe.utils.datatable"); + +frappe.utils.datatable.get_translations = function () { + let translations = {}; + translations[frappe.boot.lang] = { + "Sort Ascending": __("Sort Ascending"), + "Sort Descending": __("Sort Descending"), + "Reset sorting": __("Reset sorting"), + "Remove column": __("Remove column"), + "No Data": __("No Data"), + "{count} cells copied": { + "1": __("{count} cell copied"), + "default": __("{count} cells copied") + }, + "{count} rows selected": { + "1": __("{count} row selected"), + "default": __("{count} rows selected") + } + }; + + return translations; +} \ No newline at end of file diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 920a252b56..7ba0a0228f 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -105,7 +105,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { this.toggle_nothing_to_show(true); return; } - + let route_options = {}; route_options = Object.assign(route_options, frappe.route_options); @@ -849,6 +849,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { columns: columns, data: data, inlineFilters: true, + language: frappe.boot.lang, + translations: frappe.utils.datatable.get_translations(), treeView: this.tree_report, layout: 'fixed', cellHeight: 33, diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index c70c64be0e..6d8e281793 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -284,6 +284,8 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { columns: this.columns, data: this.get_data(values), getEditor: this.get_editing_object.bind(this), + language: frappe.boot.lang, + translations: frappe.utils.datatable.get_translations(), checkboxColumn: true, inlineFilters: true, cellHeight: 35, From a3c5c08cff91139effcf1c416e38c6ea18a42519 Mon Sep 17 00:00:00 2001 From: hrwx Date: Thu, 6 Jan 2022 13:07:04 +0530 Subject: [PATCH 60/79] chore: bump datatable version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1d062315a0..1bbedb33fc 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "express": "^4.17.1", "fast-deep-equal": "^2.0.1", "frappe-charts": "^2.0.0-rc13", - "frappe-datatable": "^1.15.4", + "frappe-datatable": "^1.16.0", "frappe-gantt": "^0.5.0", "fuse.js": "^3.4.6", "highlight.js": "^10.4.1", From 0a8b5fe7857c875c6b08b728887d93ba96dd2cd4 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 6 Jan 2022 13:38:48 +0530 Subject: [PATCH 61/79] test: Fix get_field command for select fields --- cypress/support/commands.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 4fe315c372..758b3cde2b 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -193,7 +193,8 @@ Cypress.Commands.add('fill_field', (fieldname, value, fieldtype = 'Data') => { }); Cypress.Commands.add('get_field', (fieldname, fieldtype = 'Data') => { - let selector = `[data-fieldname="${fieldname}"] input:visible`; + let field_element = fieldtype === 'Select' ? 'select': 'input'; + let selector = `[data-fieldname="${fieldname}"] ${field_element}:visible`; if (fieldtype === 'Text Editor') { selector = `[data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]:visible`; From 379342d0caaed320e864343d888f24f66d36f067 Mon Sep 17 00:00:00 2001 From: Verequies Date: Tue, 22 Jun 2021 13:40:41 +1000 Subject: [PATCH 62/79] fix: Postgres Compatibility for Standard Views Core/User DocType: Add 'group_by' for PostgreSQL Signed-off-by: Verequies fix: Postgres Compatibility for Standard Views Desk/Dashboard DocType: Fix lowercase table name Signed-off-by: Verequies Desk/Page/Setup Wizard: Fix data not being commited to database Signed-off-by: Verequies Model/db_query: Fix queries with order_by and group_by for PostgreSQL Signed-off-by: Verequies Model/db_query: Fix order_by without table name for PostgreSQL Signed-off-by: Verequies --- frappe/core/doctype/user_type/user_type.py | 7 ++----- frappe/desk/doctype/dashboard/dashboard.py | 2 +- frappe/desk/page/setup_wizard/setup_wizard.py | 2 +- frappe/model/db_query.py | 14 ++++++++++++++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index c1fd678141..6a07b5a23b 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -36,11 +36,8 @@ class UserType(Document): if not self.user_doctypes: return - modules = frappe.get_all("DocType", - fields=["module"], - filters={"name": ("in", [d.document_type for d in self.user_doctypes])}, - distinct=True, - ) + modules = frappe.get_all('DocType', fields=['distinct module as module'], + filters={'name': ('in', [d.document_type for d in self.user_doctypes])}, group_by='module') self.set('user_type_modules', []) for row in modules: diff --git a/frappe/desk/doctype/dashboard/dashboard.py b/frappe/desk/doctype/dashboard/dashboard.py index 0dfd458a37..833447a00b 100644 --- a/frappe/desk/doctype/dashboard/dashboard.py +++ b/frappe/desk/doctype/dashboard/dashboard.py @@ -14,7 +14,7 @@ class Dashboard(Document): if self.is_default: # make all other dashboards non-default frappe.db.sql('''update - tabDashboard set is_default = 0 where name != %s''', self.name) + `tabDashboard` set is_default = 0 where name != %s''', self.name) if frappe.conf.developer_mode and self.is_standard: export_to_files(record_list=[['Dashboard', self.name, self.module + ' Dashboard']], record_module=self.module) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index 83a5e16009..6d7ecce462 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -388,7 +388,6 @@ def make_records(records, debug=False): # LOG every success and failure for record in records: - doctype = record.get("doctype") condition = record.get('__condition') @@ -405,6 +404,7 @@ def make_records(records, debug=False): try: doc.insert(ignore_permissions=True) + frappe.db.commit() except frappe.DuplicateEntryError as e: # print("Failed to insert duplicate {0} {1}".format(doctype, doc.name)) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 16c0d18d9f..3c5d6e81ac 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -128,6 +128,20 @@ class DatabaseQuery(object): args.fields = 'distinct ' + args.fields args.order_by = '' # TODO: recheck for alternative + if args.order_by and args.group_by: + order_field = args.order_by + + for r in (" order by ", " asc", " ASC", " desc", " DESC"): + order_field = order_field.replace(r, "") + + if not order_field in args.fields: + order_fieldm = order_field.replace("`", "") + if "." in order_fieldm: + args.fields += ", MAX(" + order_fieldm.split(".")[1] + ") as `" + order_fieldm + "`" + else: + args.fields += ", MAX(" + order_fieldm + ") as `" + order_fieldm + "`" + args.order_by = args.order_by.replace(order_field, "`" + order_fieldm + "`") + query = """select %(fields)s from %(tables)s %(conditions)s From 080b3b889f7f8bad1f85a8968cfa894cdc351a4d Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 6 Jan 2022 16:08:10 +0530 Subject: [PATCH 63/79] fix: border color fix in dark mode --- frappe/public/scss/desk/form.scss | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/scss/desk/form.scss b/frappe/public/scss/desk/form.scss index 7e7d6170c9..f56d9da59a 100644 --- a/frappe/public/scss/desk/form.scss +++ b/frappe/public/scss/desk/form.scss @@ -54,7 +54,7 @@ .form-section.card-section, .form-dashboard-section { - border-bottom: 1px solid var(--gray-200); + border-bottom: 1px solid var(--border-color); padding: var(--padding-xs); } @@ -316,12 +316,12 @@ .form-tabs-list { padding-left: var(--padding-xs); - border-bottom: 1px solid var(--gray-200); + border-bottom: 1px solid var(--border-color); .form-tabs { .nav-item { .nav-link { - color: var(--gray-700); + color: var(--text-muted); padding: var(--padding-md) 0; margin: 0 var(--margin-md); From 24ac11c98fbc5c6a9acd5abc7dd912ac5f7766b7 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Thu, 6 Jan 2022 15:43:17 +0100 Subject: [PATCH 64/79] fix: update yarn.lock --- yarn.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index ba58f6b719..a7cbf0cf60 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1601,10 +1601,10 @@ frappe-charts@^2.0.0-rc13: resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc13.tgz#fdb251d7ae311c41e38f90a3ae108070ec6b9072" integrity sha512-Bv7IfllIrjRbKWHn5b769dOSenqdBixAr6m5kurf8ZUOJSLOgK4HOXItJ7BA8n9PvviH9/k5DaloisjLM2Bm1w== -frappe-datatable@^1.15.4: - version "1.15.4" - resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.15.4.tgz#dc2e5e5d8a0a7cb8ee658f2d39966af1d4405401" - integrity sha512-eW3upPvverm1GNBL4+IcPDvjm5xbJc5ZXW8TYEUZt/QQ2W75K/T6736pSzi9D6mX9sn3BtZ7Ige7MS45SGrgzQ== +frappe-datatable@^1.16.0: + version "1.16.0" + resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.16.0.tgz#822dbcaaf48e5171f47ce2909da88e9a31bb2cbc" + integrity sha512-FkeHcaxxz4+BQLhwiegk94602PlnWNG4LiBPehgKdEL+OpJcwl8oNKWb/wPNw9lgW25u0LaaQwq/11sw7mnEbA== dependencies: hyperlist "^1.0.0-beta" lodash "^4.17.5" From f04be3f8768a2df47f843e294aa2f228995458cd Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Thu, 6 Jan 2022 15:44:29 +0100 Subject: [PATCH 65/79] fix: sider --- frappe/public/js/frappe/utils/datatable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/datatable.js b/frappe/public/js/frappe/utils/datatable.js index e07b86df94..ec82d256f1 100644 --- a/frappe/public/js/frappe/utils/datatable.js +++ b/frappe/public/js/frappe/utils/datatable.js @@ -19,4 +19,4 @@ frappe.utils.datatable.get_translations = function () { }; return translations; -} \ No newline at end of file +}; From ea5695700ca92a187dee34f16e0079c74a492ea3 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Thu, 6 Jan 2022 15:52:05 +0100 Subject: [PATCH 66/79] feat: add german translations for datatable menu --- frappe/translations/de.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index 5b45d8c217..1dc542f55d 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -4700,3 +4700,7 @@ Value cannot be negative for {0}: {1},Der Wert kann für {0} nicht negativ sein: Negative Value,Negativer Wert, Authentication failed while receiving emails from Email Account: {0}.,Die Authentifizierung ist beim Empfang von E-Mails vom E-Mail-Konto fehlgeschlagen: {0}., Message from server: {0},Nachricht vom Server: {0}, +Reset sorting,Sortierung zurücksetzen, +Sort Ascending,Aufsteigend sortieren, +Sort Descending,Absteigend sortieren, +Remove column,Spalte entfernen, From 40ba1ac9ba3909ffff30c03029e70a43b520c483 Mon Sep 17 00:00:00 2001 From: Conor Date: Sun, 19 Dec 2021 12:16:34 -0600 Subject: [PATCH 67/79] fix: Postgres Compatibility * Handle inconsistencies in type handling in DatabaseQuery & Database APIs * Update incompatible queries with frappe.qb notation * Fixed use cases discovered by failing ERPNext CI tests fix: db independent syntax for user_type fix: handle postgres datetime values feat: add ability to auto commit on db inserts feat: add ability to escape underscore in postgres fix: handle missing data in test runner bootstrapping fix: db independent syntax for queries fix: refactor to use qb fix: update cache for language fix: use pluck in email_queue Co-authored-by: gavin fix: don't auto insert on tests for make_property_setter fix: remove auto_commit in custom_field insertion fix: remove auto_commit functionality fix: review comments fix: revert link validation fix: style suggestion for readability Co-authored-by: gavin fix: revert .lower() in link validation fix: add rollback for setup_wizard Revert "fix: add rollback for setup_wizard" This reverts commit 83b3b0913db17718ccd5edae01858cff15603829. Revert "feat: add ability to escape underscore in postgres" This reverts commit 8ed9c2aa3306438e94bb813f60e65b416d0b947b. fix: more concise representation of order fields Co-authored-by: gavin --- frappe/core/doctype/user_type/user_type.py | 12 +++++-- frappe/database/postgres/database.py | 8 ++++- .../desk/doctype/bulk_update/bulk_update.py | 3 +- frappe/desk/doctype/dashboard/dashboard.py | 5 +-- frappe/desk/page/setup_wizard/setup_wizard.py | 3 +- frappe/desk/treeview.py | 5 +-- .../email/doctype/email_queue/email_queue.py | 32 +++++++------------ frappe/model/base_document.py | 4 ++- frappe/model/db_query.py | 13 ++++---- frappe/test_runner.py | 5 ++- frappe/tests/test_website.py | 1 + frappe/utils/data.py | 3 ++ frappe/utils/make_random.py | 10 ++++-- frappe/utils/nestedset.py | 14 ++++---- frappe/utils/response.py | 2 +- .../doctype/blog_post/test_blog_post.py | 5 ++- 16 files changed, 75 insertions(+), 50 deletions(-) diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 6a07b5a23b..5775d499bd 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -36,13 +36,19 @@ class UserType(Document): if not self.user_doctypes: return - modules = frappe.get_all('DocType', fields=['distinct module as module'], - filters={'name': ('in', [d.document_type for d in self.user_doctypes])}, group_by='module') + DocType = frappe.qb.DocType("DocType") + + document_types = [d.document_type for d in self.user_doctypes] or [''] + modules = (frappe.qb.from_(DocType) + .select(DocType.module) + .where(DocType.name.isin(document_types)) + .groupby(DocType.module) + .distinct()).run() self.set('user_type_modules', []) for row in modules: self.append('user_type_modules', { - 'module': row.module + 'module': row[0] }) def validate_document_type_limit(self): diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 3cea1440cf..795d36eaeb 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -74,9 +74,15 @@ class PostgresDatabase(Database): return conn def escape(self, s, percent=True): - """Excape quotes and percent in given string.""" + """Escape quotes and percent in given string.""" if isinstance(s, bytes): s = s.decode('utf-8') + + # MariaDB's driver treats None as an empty string + # So Postgres should do the same + + if s is None: + s = '' if percent: s = s.replace("%", "%%") diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index b512ca175c..a0523d90cd 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -7,6 +7,7 @@ from frappe.model.document import Document from frappe import _ from frappe.utils import cint + class BulkUpdate(Document): pass @@ -22,7 +23,7 @@ def update(doctype, field, value, condition='', limit=500): frappe.throw(_('; not allowed in condition')) docnames = frappe.db.sql_list( - '''select name from `tab{0}`{1} limit 0, {2}'''.format(doctype, condition, limit) + '''select name from `tab{0}`{1} limit {2} offset 0'''.format(doctype, condition, limit) ) data = {} data[field] = value diff --git a/frappe/desk/doctype/dashboard/dashboard.py b/frappe/desk/doctype/dashboard/dashboard.py index 833447a00b..55044cda4b 100644 --- a/frappe/desk/doctype/dashboard/dashboard.py +++ b/frappe/desk/doctype/dashboard/dashboard.py @@ -13,8 +13,9 @@ class Dashboard(Document): def on_update(self): if self.is_default: # make all other dashboards non-default - frappe.db.sql('''update - `tabDashboard` set is_default = 0 where name != %s''', self.name) + DashBoard = frappe.qb.DocType("Dashboard") + + frappe.qb.update(DashBoard).set(DashBoard.is_default, 0).where(DashBoard.name != self.name).run() if frappe.conf.developer_mode and self.is_standard: export_to_files(record_list=[['Dashboard', self.name, self.module + ' Dashboard']], record_module=self.module) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index 6d7ecce462..b42d8c58b7 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -151,7 +151,7 @@ def update_system_settings(args): system_settings = frappe.get_doc("System Settings", "System Settings") system_settings.update({ "country": args.get("country"), - "language": get_language_code(args.get("language")), + "language": get_language_code(args.get("language")) or 'en', "time_zone": args.get("timezone"), "float_precision": 3, 'date_format': frappe.db.get_value("Country", args.get("country"), "date_format"), @@ -417,6 +417,7 @@ def make_records(records, debug=False): raise except Exception as e: + frappe.db.rollback() exception = record.get('__exception') if exception: config = _dict(exception) diff --git a/frappe/desk/treeview.py b/frappe/desk/treeview.py index f40c135653..7e3efb5d48 100644 --- a/frappe/desk/treeview.py +++ b/frappe/desk/treeview.py @@ -4,6 +4,7 @@ import frappe from frappe import _ + @frappe.whitelist() def get_all_nodes(doctype, label, parent, tree_method, **filters): '''Recursively gets all data from tree nodes''' @@ -40,8 +41,8 @@ def get_children(doctype, parent='', **filters): def _get_children(doctype, parent='', ignore_permissions=False): parent_field = 'parent_' + doctype.lower().replace(' ', '_') - filters = [['ifnull(`{0}`,"")'.format(parent_field), '=', parent], - ['docstatus', '<' ,'2']] + filters = [["ifnull(`{0}`,'')".format(parent_field), '=', parent], + ['docstatus', '<' ,2]] meta = frappe.get_meta(doctype) diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index d89a3d83be..9730004065 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -475,28 +475,20 @@ class QueueBuilder: if self._unsubscribed_user_emails is not None: return self._unsubscribed_user_emails - all_ids = tuple(set(self.recipients + self.cc)) + all_ids = list(set(self.recipients + self.cc)) - unsubscribed = frappe.db.sql_list(''' - SELECT - distinct email - from - `tabEmail Unsubscribe` - where - email in %(all_ids)s - and ( - ( - reference_doctype = %(reference_doctype)s - and reference_name = %(reference_name)s - ) - or global_unsubscribe = 1 - ) - ''', { - 'all_ids': all_ids, - 'reference_doctype': self.reference_doctype, - 'reference_name': self.reference_name, - }) + EmailUnsubscribe = frappe.qb.DocType("Email Unsubscribe") + unsubscribed = (frappe.qb.from_(EmailUnsubscribe) + .select(EmailUnsubscribe.email) + .where(EmailUnsubscribe.email.isin(all_ids) & + ( + ( + (EmailUnsubscribe.reference_doctype == self.reference_doctype) & (EmailUnsubscribe.reference_name == self.reference_name) + ) | EmailUnsubscribe.global_unsubscribe == 1 + ) + ).distinct() + ).run(pluck=True) self._unsubscribed_user_emails = unsubscribed or [] return self._unsubscribed_user_emails diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 1fd3784fcc..d24c1722a8 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -771,7 +771,9 @@ class BaseDocument(object): else: self_value = self.get_value(key) - + # Postgres stores values as `datetime.time`, MariaDB as `timedelta` + if isinstance(self_value, datetime.timedelta) and isinstance(db_value, datetime.time): + db_value = datetime.timedelta(hours=db_value.hour, minutes=db_value.minute, seconds=db_value.second, microseconds=db_value.microsecond) if self_value != db_value: frappe.throw(_("Not allowed to change {0} after submission").format(df.label), frappe.UpdateAfterSubmitError) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 3c5d6e81ac..73273917f7 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -127,19 +127,18 @@ class DatabaseQuery(object): if self.distinct: args.fields = 'distinct ' + args.fields args.order_by = '' # TODO: recheck for alternative - + + # Postgres requires any field that appears in the select clause to also + # appear in the order by and group by clause if args.order_by and args.group_by: order_field = args.order_by for r in (" order by ", " asc", " ASC", " desc", " DESC"): order_field = order_field.replace(r, "") - if not order_field in args.fields: - order_fieldm = order_field.replace("`", "") - if "." in order_fieldm: - args.fields += ", MAX(" + order_fieldm.split(".")[1] + ") as `" + order_fieldm + "`" - else: - args.fields += ", MAX(" + order_fieldm + ") as `" + order_fieldm + "`" + if order_field not in args.fields: + order_fieldm = order_field.replace("`", '"') + args.fields += f", MAX({order_fieldm}) as `{order_fieldm}`" args.order_by = args.order_by.replace(order_field, "`" + order_fieldm + "`") query = """select %(fields)s diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 5f26842be2..1839f15ae8 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -335,7 +335,10 @@ def make_test_records_for_doctype(doctype, verbose=0, force=False): frappe.local.test_objects[doctype] += test_module._make_test_records(verbose) elif hasattr(test_module, "test_records"): - frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force) + if doctype in frappe.local.test_objects: + frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force) + else: + frappe.local.test_objects[doctype] = make_test_objects(doctype, test_module.test_records, verbose, force) else: test_records = frappe.get_test_records(doctype) diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index 992d876243..e40a07c0ec 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -197,6 +197,7 @@ class TestWebsite(unittest.TestCase): frappe.cache().delete_key('app_hooks') def test_printview_page(self): + frappe.db.value_cache[('DocType', 'Language', 'name')] = (('Language',),) content = get_response_content('/Language/ru') self.assertIn('
', content) self.assertIn('
Language
', content) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index de0242df07..cdd4d3d277 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -98,6 +98,9 @@ def get_timedelta(time: Optional[str] = None) -> Optional[datetime.timedelta]: def to_timedelta(time_str): from dateutil import parser + if isinstance(time_str, datetime.time): + time_str = str(time_str) + if isinstance(time_str, str): t = parser.parse(time_str) return datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second, microseconds=t.microsecond) diff --git a/frappe/utils/make_random.py b/frappe/utils/make_random.py index 2ebabb78f9..bb4395b626 100644 --- a/frappe/utils/make_random.py +++ b/frappe/utils/make_random.py @@ -35,9 +35,13 @@ def get_random(doctype, filters=None, doc=False): condition = " where " + " and ".join(condition) else: condition = "" - - out = frappe.db.sql("""select name from `tab%s` %s - order by RAND() limit 0,1""" % (doctype, condition)) + + out = frappe.db.multisql({ + 'mariadb': """select name from `tab%s` %s + order by RAND() limit 1 offset 0""" % (doctype, condition), + 'postgres': """select name from `tab%s` %s + order by RANDOM() limit 1 offset 0""" % (doctype, condition) + }) out = out and out[0][0] or None diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index 5c4beda71e..1fc2d7779a 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -16,6 +16,9 @@ import frappe from frappe import _ from frappe.model.document import Document from frappe.query_builder import DocType, Order +from pypika import CustomFunction + +Replace = CustomFunction('REPLACE', ['input', 'substr_to_replace', 'substr_to_replace_with']) class NestedSetRecursionError(frappe.ValidationError): pass class NestedSetMultipleRootsError(frappe.ValidationError): pass @@ -88,7 +91,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab{0}` - where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0] + where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0] validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt) @@ -107,8 +110,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab%s` - where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0] - + where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0] # set parent lft, rgt frappe.db.sql("""update `tab{0}` set rgt = rgt + %s @@ -134,6 +136,7 @@ def update_move_node(doc, parent_field): frappe.db.sql("""update `tab{0}` set lft = -lft + %s, rgt = -rgt + %s where lft < 0""".format(doc.doctype), (new_diff, new_diff)) + @frappe.whitelist() def rebuild_tree(doctype, parent_field): """ @@ -148,13 +151,12 @@ def rebuild_tree(doctype, parent_field): right = 1 table = DocType(doctype) column = getattr(table, parent_field) - result = ( frappe.qb.from_(table) .where( (column == "") | (column.isnull()) ) - .orderby(table.name, order=Order.asc) + .orderby(Replace(table.name, '_', ''), order=Order.asc) .select(table.name) ).run() @@ -177,7 +179,7 @@ def rebuild_node(doctype, parent, left, parent_field): column = getattr(table, parent_field) result = ( - frappe.qb.from_(table).where(column == parent).select(table.name) + frappe.qb.from_(table).where(column == parent).select(table.name).orderby(Replace(table.name, '_', ''), order=Order.asc) ).run() for r in result: diff --git a/frappe/utils/response.py b/frappe/utils/response.py index 104c48527c..f6ad91dbd2 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -125,7 +125,7 @@ def json_handler(obj): # serialize date import collections.abc - if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime)): + if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime, datetime.time)): return str(obj) elif isinstance(obj, decimal.Decimal): diff --git a/frappe/website/doctype/blog_post/test_blog_post.py b/frappe/website/doctype/blog_post/test_blog_post.py index 7a09d0a3ca..d649d25f7e 100644 --- a/frappe/website/doctype/blog_post/test_blog_post.py +++ b/frappe/website/doctype/blog_post/test_blog_post.py @@ -58,15 +58,18 @@ class TestBlogPost(unittest.TestCase): category_page_link = list(soup.find_all('a', href=re.compile(blog.blog_category)))[0] category_page_url = category_page_link["href"] + cached_value = frappe.db.value_cache[('DocType', 'Blog Post', 'name')] + frappe.db.value_cache[('DocType', 'Blog Post', 'name')] = (('Blog Post',),) + # Visit the category page (by following the link found in above stage) set_request(path=category_page_url) category_page_response = get_response() category_page_html = frappe.safe_decode(category_page_response.get_data()) - # Category page should contain the blog post title self.assertIn(blog.title, category_page_html) # Cleanup + frappe.db.value_cache[('DocType', 'Blog Post', 'name')] = cached_value frappe.delete_doc("Blog Post", blog.name) frappe.delete_doc("Blog Category", blog.blog_category) From 1c40881c411f8d67dc399a6a4b2ded7eed1168d4 Mon Sep 17 00:00:00 2001 From: Conor Date: Wed, 29 Dec 2021 16:49:23 -0600 Subject: [PATCH 68/79] fix: refactor postgres arg prep into its own function --- frappe/model/db_query.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 73273917f7..d296c24362 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -130,16 +130,8 @@ class DatabaseQuery(object): # Postgres requires any field that appears in the select clause to also # appear in the order by and group by clause - if args.order_by and args.group_by: - order_field = args.order_by - - for r in (" order by ", " asc", " ASC", " desc", " DESC"): - order_field = order_field.replace(r, "") - - if order_field not in args.fields: - order_fieldm = order_field.replace("`", '"') - args.fields += f", MAX({order_fieldm}) as `{order_fieldm}`" - args.order_by = args.order_by.replace(order_field, "`" + order_fieldm + "`") + if frappe.db.db_type == 'postgres' and args.order_by and args.group_by: + args = self.prepare_select_args(args) query = """select %(fields)s from %(tables)s @@ -214,6 +206,19 @@ class DatabaseQuery(object): return args + def prepare_select_args(self, args): + order_field = args.order_by + + for r in (" order by ", " asc", " ASC", " desc", " DESC"): + order_field = order_field.replace(r, "") + + if order_field not in args.fields: + order_fieldm = order_field.replace("`", '"') + args.fields += f", MAX({order_fieldm}) as `{order_fieldm}`" + args.order_by = args.order_by.replace(order_field, "`" + order_fieldm + "`") + + return args + def parse_args(self): """Convert fields and filters from strings to list, dicts""" if isinstance(self.fields, str): From 55392b64e7bbadcacee52959fd2b5d4d9614ef02 Mon Sep 17 00:00:00 2001 From: Conor Date: Wed, 29 Dec 2021 17:10:06 -0600 Subject: [PATCH 69/79] fix: Revert changes made to fix collation differences Due to collation differences in MariaDB and Postgres, tests gave inconsistent results. This was to be handled in tests alone instead of the application. It was unnecessary. Collation changes should be made at DBMS config level only. Accomodating for those in the application will unnecessarily degrade performance for everyone. Other changes: * use pluck in user_type * revert ordering in nestedset * revert parsing in order_field * use preferred APIs & styling Co-authored-by: gavin --- frappe/core/doctype/user_type/user_type.py | 21 ++++++---------- frappe/desk/doctype/dashboard/dashboard.py | 29 ++++++++++++++-------- frappe/model/db_query.py | 7 ++++-- frappe/utils/nestedset.py | 13 ++++------ 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 5775d499bd..661ac932e7 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -36,20 +36,15 @@ class UserType(Document): if not self.user_doctypes: return - DocType = frappe.qb.DocType("DocType") + modules = frappe.get_all("DocType", + filters={"name": ("in", [d.document_type for d in self.user_doctypes])}, + distinct=True, + pluck="module", + ) - document_types = [d.document_type for d in self.user_doctypes] or [''] - modules = (frappe.qb.from_(DocType) - .select(DocType.module) - .where(DocType.name.isin(document_types)) - .groupby(DocType.module) - .distinct()).run() - - self.set('user_type_modules', []) - for row in modules: - self.append('user_type_modules', { - 'module': row[0] - }) + self.set("user_type_modules", []) + for module in modules: + self.append("user_type_modules", {"module": module}) def validate_document_type_limit(self): limit = frappe.conf.get('user_type_doctype_limit', {}).get(frappe.scrub(self.name)) diff --git a/frappe/desk/doctype/dashboard/dashboard.py b/frappe/desk/doctype/dashboard/dashboard.py index 55044cda4b..ac62796dc2 100644 --- a/frappe/desk/doctype/dashboard/dashboard.py +++ b/frappe/desk/doctype/dashboard/dashboard.py @@ -1,24 +1,33 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and contributors +# Copyright (c) 2022, Frappe Technologies and contributors # License: MIT. See LICENSE -from frappe.model.document import Document -from frappe.modules.export_file import export_to_files -from frappe.config import get_modules_from_all_apps_for_user +import json + import frappe from frappe import _ -import json +from frappe.config import get_modules_from_all_apps_for_user +from frappe.model.document import Document +from frappe.modules.export_file import export_to_files +from frappe.query_builder import DocType + class Dashboard(Document): def on_update(self): if self.is_default: # make all other dashboards non-default - DashBoard = frappe.qb.DocType("Dashboard") - - frappe.qb.update(DashBoard).set(DashBoard.is_default, 0).where(DashBoard.name != self.name).run() + DashBoard = DocType("Dashboard") + + frappe.qb.update(DashBoard).set( + DashBoard.is_default, 0 + ).where( + DashBoard.name != self.name + ).run() if frappe.conf.developer_mode and self.is_standard: - export_to_files(record_list=[['Dashboard', self.name, self.module + ' Dashboard']], record_module=self.module) + export_to_files( + record_list=[["Dashboard", self.name, f"{self.module} Dashboard"]], + record_module=self.module + ) def validate(self): if not frappe.conf.developer_mode and self.is_standard: diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index d296c24362..1bb42f9c97 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -213,8 +213,11 @@ class DatabaseQuery(object): order_field = order_field.replace(r, "") if order_field not in args.fields: - order_fieldm = order_field.replace("`", '"') - args.fields += f", MAX({order_fieldm}) as `{order_fieldm}`" + order_fieldm = order_field.replace("`", "") + if "." in order_fieldm: + args.fields += ", MAX(" + order_fieldm.split(".")[1] + ") as `" + order_fieldm + "`" + else: + args.fields += ", MAX(" + order_fieldm + ") as `" + order_fieldm + "`" args.order_by = args.order_by.replace(order_field, "`" + order_fieldm + "`") return args diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index 1fc2d7779a..34322092dc 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE # Tree (Hierarchical) Nested Set Model (nsm) @@ -16,9 +16,6 @@ import frappe from frappe import _ from frappe.model.document import Document from frappe.query_builder import DocType, Order -from pypika import CustomFunction - -Replace = CustomFunction('REPLACE', ['input', 'substr_to_replace', 'substr_to_replace_with']) class NestedSetRecursionError(frappe.ValidationError): pass class NestedSetMultipleRootsError(frappe.ValidationError): pass @@ -91,7 +88,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab{0}` - where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0] + where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0] validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt) @@ -110,7 +107,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab%s` - where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0] + where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0] # set parent lft, rgt frappe.db.sql("""update `tab{0}` set rgt = rgt + %s @@ -156,7 +153,7 @@ def rebuild_tree(doctype, parent_field): .where( (column == "") | (column.isnull()) ) - .orderby(Replace(table.name, '_', ''), order=Order.asc) + .orderby(table.name, order=Order.asc) .select(table.name) ).run() @@ -179,7 +176,7 @@ def rebuild_node(doctype, parent, left, parent_field): column = getattr(table, parent_field) result = ( - frappe.qb.from_(table).where(column == parent).select(table.name).orderby(Replace(table.name, '_', ''), order=Order.asc) + frappe.qb.from_(table).where(column == parent).select(table.name) ).run() for r in result: From d9b571cf43727d4ca6b4ff514f4199258cc0e3aa Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 7 Jan 2022 11:25:06 +0530 Subject: [PATCH 70/79] refactor(minor): DatabaseQuery.prepare_select_args * Used regex to replace instead of re-iteration (for-loop) * Used f-strings for better readability * Renamed and re-ordered vars, code block --- frappe/model/db_query.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 1bb42f9c97..f973cbbb8a 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -127,8 +127,8 @@ class DatabaseQuery(object): if self.distinct: args.fields = 'distinct ' + args.fields args.order_by = '' # TODO: recheck for alternative - - # Postgres requires any field that appears in the select clause to also + + # Postgres requires any field that appears in the select clause to also # appear in the order by and group by clause if frappe.db.db_type == 'postgres' and args.order_by and args.group_by: args = self.prepare_select_args(args) @@ -207,18 +207,15 @@ class DatabaseQuery(object): return args def prepare_select_args(self, args): - order_field = args.order_by - - for r in (" order by ", " asc", " ASC", " desc", " DESC"): - order_field = order_field.replace(r, "") + order_field = re.sub(r"\ order\ by\ |\ asc|\ ASC|\ desc|\ DESC", "", args.order_by) if order_field not in args.fields: - order_fieldm = order_field.replace("`", "") - if "." in order_fieldm: - args.fields += ", MAX(" + order_fieldm.split(".")[1] + ") as `" + order_fieldm + "`" - else: - args.fields += ", MAX(" + order_fieldm + ") as `" + order_fieldm + "`" - args.order_by = args.order_by.replace(order_field, "`" + order_fieldm + "`") + extracted_column = order_column = order_field.replace("`", "") + if "." in extracted_column: + extracted_column = extracted_column.split(".")[1] + + args.fields += f", MAX({extracted_column}) as `{order_column}`" + args.order_by = args.order_by.replace(order_field, f"`{order_column}`") return args From 6afa5c8da3c7d0e74425f572e39f042e76ae6d19 Mon Sep 17 00:00:00 2001 From: Ahmad Date: Thu, 6 Jan 2022 17:33:47 +0500 Subject: [PATCH 71/79] fix: email_string parsing when pulling email --- frappe/core/doctype/communication/communication.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 96c8f271d9..5a1f03cd3d 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -12,7 +12,7 @@ from frappe.core.utils import get_parent_doc from frappe.utils.bot import BotReply from frappe.utils import parse_addr, split_emails from frappe.core.doctype.comment.comment import update_comment_in_doc -from email.utils import parseaddr +from email.utils import getaddresses from urllib.parse import unquote from frappe.utils.user import is_system_user from frappe.contacts.doctype.contact.contact import get_contact_name @@ -372,10 +372,9 @@ def get_contacts(email_strings, auto_create_contact=False): for email_string in email_strings: if email_string: - for email in email_string.split(","): - parsed_email = parseaddr(email)[1] - if parsed_email: - email_addrs.append(parsed_email) + result = getaddresses(email_string) + for email in result: + email_addrs.append(email[1]) contacts = [] for email in email_addrs: From 9ebfc1ced9cfb7643560858553e59243eb128ac4 Mon Sep 17 00:00:00 2001 From: Ahmad Date: Thu, 6 Jan 2022 18:32:57 +0500 Subject: [PATCH 72/79] refactor: communication.py --- frappe/core/doctype/communication/communication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 5a1f03cd3d..1ab07d92e4 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -372,7 +372,7 @@ def get_contacts(email_strings, auto_create_contact=False): for email_string in email_strings: if email_string: - result = getaddresses(email_string) + result = getaddresses([email_string]) for email in result: email_addrs.append(email[1]) From 3d474b592721fac37bb06ce4a520a65fa828f9bb Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 10 Jan 2022 13:45:18 +0530 Subject: [PATCH 73/79] fix: drop py3.7 support for v14 --- .github/workflows/docs-checker.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml index 02a01bf4e4..5e91063698 100644 --- a/.github/workflows/docs-checker.yml +++ b/.github/workflows/docs-checker.yml @@ -12,7 +12,7 @@ jobs: - name: 'Setup Environment' uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8 - name: 'Clone repo' uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 71f7f1fab2..c7ed9f1aea 100644 --- a/setup.py +++ b/setup.py @@ -57,5 +57,5 @@ setup( { 'clean': CleanCommand }, - python_requires='>=3.7' + python_requires='>=3.8' ) From d1852242a0acb37c799d00c2b9fc8ca6d71b3587 Mon Sep 17 00:00:00 2001 From: Pruthvi Patel Date: Mon, 10 Jan 2022 15:52:25 +0530 Subject: [PATCH 74/79] fix: limit in `Document.get` --- frappe/model/base_document.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index eeef552a8a..6353eff95d 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -1008,15 +1008,12 @@ def _filter(data, filters, limit=None): _filters[f] = fval for d in data: - add = True for f, fval in _filters.items(): if not frappe.compare(getattr(d, f, None), fval[0], fval[1]): - add = False break - - if add: + else: out.append(d) - if limit and (len(out)-1)==limit: + if limit and len(out) >= limit: break return out From 3d77f544fdaccac472c5ea72a9c603554aacc469 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 11 Jan 2022 10:09:14 +0530 Subject: [PATCH 75/79] fix(datetime): Fix side-effect due to `__run_link_triggers` - `__run_link_triggers` should only affect link fields - Removed usage of `__run_link_triggers` from base control and introduce `force_set_value` for more explicit action. --- .../public/js/frappe/form/controls/base_control.js | 14 +++++++------- frappe/public/js/frappe/form/form.js | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index 7af0705e78..ce871c50cb 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -148,8 +148,9 @@ frappe.ui.form.Control = class BaseControl { return this.doc[this.df.fieldname]; } } - set_value(value) { - return this.validate_and_set_in_model(value); + + set_value(value, force_set_value=false) { + return this.validate_and_set_in_model(value, null, force_set_value); } parse_validate_and_set_in_model(value, e) { if(this.parse) { @@ -157,12 +158,11 @@ frappe.ui.form.Control = class BaseControl { } return this.validate_and_set_in_model(value, e); } - validate_and_set_in_model(value, e) { - var me = this; - let force_value_set = (this.doc && this.doc.__run_link_triggers); - let is_value_same = (this.get_model_value() === value); + validate_and_set_in_model(value, e, force_set_value=false) { + const me = this; + const is_value_same = (this.get_model_value() === value); - if (this.inside_change_event || (!force_value_set && is_value_same)) { + if (this.inside_change_event || (is_value_same && !force_set_value)) { return Promise.resolve(); } diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 57e3f576a1..1459b38df6 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -983,7 +983,7 @@ frappe.ui.form.Form = class FrappeForm { $.each(this.fields_dict, function(fieldname, field) { if (field.df.fieldtype=="Link" && this.doc[fieldname]) { // triggers add fetch, sets value in model and runs triggers - field.set_value(this.doc[fieldname]); + field.set_value(this.doc[fieldname], true); } }); From a87899c2155a0c2576d551a31aa5f6535be6930b Mon Sep 17 00:00:00 2001 From: Pruthvi Patel Date: Tue, 11 Jan 2022 11:12:53 +0530 Subject: [PATCH 76/79] test: add test for limit --- frappe/tests/test_document.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 29cec8b230..34a1dd070c 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -252,3 +252,8 @@ class TestDocument(unittest.TestCase): 'currency': 100000 }) self.assertEquals(d.get_formatted('currency', currency='INR', format="#,###.##"), '₹ 100,000.00') + + def test_limit_for_get(self): + doc = frappe.get_doc("DocType", "DocType") + # assuming DocType has more that 3 Data fields + self.assertEquals(len(doc.get("fields", filters={"fieldtype": "Data"}, limit=3)), 3) \ No newline at end of file From 58f7b02c307db854386569612c87045ca5feb52c Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Fri, 7 Jan 2022 18:06:53 -0600 Subject: [PATCH 77/79] fix: add test_for prepare_select_args --- frappe/tests/test_db_query.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 48e97d5bb0..5cd6690209 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -1,6 +1,8 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest +import frappe +import datetime +import unittest from frappe.model.db_query import DatabaseQuery from frappe.desk.reportview import get_filters_cond @@ -380,6 +382,22 @@ class TestReportview(unittest.TestCase): owners = DatabaseQuery("DocType").execute(filters={"name": "DocType"}, pluck="owner") self.assertEqual(owners, ["Administrator"]) + def test_prepare_select_args(self): + # frappe.get_all inserts modified field into order_by clause + # test to make sure this is inserted into select field when postgres + doctypes = frappe.get_all("DocType", + filters={"docstatus": 0, "document_type": ("!=", "")}, + group_by="document_type", + fields=["document_type", "sum(is_submittable) as is_submittable"], + limit=1, + as_list=True, + ) + if frappe.conf.db_type == "mariadb": + self.assertTrue(len(doctypes[0]) == 2) + else: + self.assertTrue(len(doctypes[0]) == 3) + self.assertTrue(isinstance(doctypes[0][2], datetime.datetime)) + def test_column_comparison(self): """Test DatabaseQuery.execute to test column comparison """ From 645fde2e675daf0fd7e13ad44a051a5453c2577c Mon Sep 17 00:00:00 2001 From: Summayya Hashmani <58825865+sumaiya2908@users.noreply.github.com> Date: Tue, 11 Jan 2022 13:41:38 +0530 Subject: [PATCH 78/79] fix: email fetch error for imaplib (#15581) * fix: remove maxline for imaplib * refactor: remove redundant line Co-authored-by: Summayya --- frappe/email/receive.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 4f4ed6d48e..dd64d0df80 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -27,11 +27,7 @@ from frappe.utils.html_utils import clean_email_html # fix due to a python bug in poplib that limits it to 2048 poplib._MAXLINE = 20480 -imaplib._MAXLINE = 20480 -# fix due to a python bug in poplib that limits it to 2048 -poplib._MAXLINE = 20480 -imaplib._MAXLINE = 20480 class EmailSizeExceededError(frappe.ValidationError): pass From 1d1ed171a1fe84c2e7aba6f61f48817b79243d34 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 11 Jan 2022 13:47:27 +0530 Subject: [PATCH 79/79] fix: Pass date object instead of string for selectDate to work "Today" button was not working because get_now_date was returning string --- frappe/public/js/frappe/form/controls/date.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index 78eb3832cc..7ad1887d62 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -10,14 +10,16 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat this.set_t_for_today(); } set_formatted_input(value) { + if (value === "Today") { + value = this.get_now_date(); + } + super.set_formatted_input(value); if (this.timepicker_only) return; if (!this.datepicker) return; if (!value) { this.datepicker.clear(); return; - } else if (value === "Today") { - value = this.get_now_date(); } let should_refresh = this.last_value && this.last_value !== value; @@ -78,7 +80,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat } get_start_date() { - return new Date(this.get_now_date()); + return this.get_now_date(); } set_datepicker() { @@ -117,7 +119,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat this.datepicker.update('position', position); } get_now_date() { - return frappe.datetime.convert_to_system_tz(frappe.datetime.now_date(true)); + return frappe.datetime.convert_to_system_tz(frappe.datetime.now_date(true), false).toDate(); } set_t_for_today() { var me = this;