From 3eb08b0efc40fbb8fea8c394b921558b5ecdc286 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 27 Jan 2022 12:18:33 +0530 Subject: [PATCH 01/68] fix: Added tabs for different sections --- frappe/desk/doctype/workspace/workspace.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/desk/doctype/workspace/workspace.json b/frappe/desk/doctype/workspace/workspace.json index 211029dfcf..960ae215e7 100644 --- a/frappe/desk/doctype/workspace/workspace.json +++ b/frappe/desk/doctype/workspace/workspace.json @@ -41,7 +41,7 @@ "collapsible": 1, "collapsible_depends_on": "charts", "fieldname": "section_break_2", - "fieldtype": "Section Break", + "fieldtype": "Tab Break", "label": "Dashboards" }, { @@ -79,14 +79,14 @@ "collapsible": 1, "collapsible_depends_on": "shortcuts", "fieldname": "section_break_15", - "fieldtype": "Section Break", + "fieldtype": "Tab Break", "label": "Shortcuts" }, { "collapsible": 1, "collapsible_depends_on": "links", "fieldname": "section_break_18", - "fieldtype": "Section Break", + "fieldtype": "Tab Break", "label": "Link Cards" }, { @@ -153,13 +153,13 @@ }, { "fieldname": "roles_section", - "fieldtype": "Section Break", + "fieldtype": "Tab Break", "label": "Roles" } ], "in_create": 1, "links": [], - "modified": "2021-12-15 19:33:00.805265", + "modified": "2022-01-27 12:06:13.111743", "modified_by": "Administrator", "module": "Desk", "name": "Workspace", @@ -189,5 +189,6 @@ } ], "sort_field": "modified", - "sort_order": "DESC" + "sort_order": "DESC", + "states": [] } \ No newline at end of file From daf9c7d4cb1422ff17f8e374a94f651e57ff4c3b Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 27 Jan 2022 12:19:16 +0530 Subject: [PATCH 02/68] fix: changed Save Customization to Save --- frappe/public/js/frappe/views/workspace/workspace.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index e28225904d..2a02cd33ce 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -376,7 +376,7 @@ frappe.views.Workspace = class Workspace { this.clear_page_actions(); page.is_editable && this.page.set_primary_action( - __("Save Customizations"), + __("Save"), () => { this.clear_page_actions(); this.save_page(page).then((saved) => { From 990b00a8084c9919ef0a9732a934e2681089e3d3 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 27 Jan 2022 12:19:53 +0530 Subject: [PATCH 03/68] fix: added shadow on main card --- frappe/public/scss/desk/desktop.scss | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss index 549ed6eee9..b471feb878 100644 --- a/frappe/public/scss/desk/desktop.scss +++ b/frappe/public/scss/desk/desktop.scss @@ -850,10 +850,16 @@ body { } } + .layout-main-section-wrapper { + margin-top: -5px; + padding-top: 5px; + } + .layout-main-section { background-color: var(--fg-color); - padding: var(--padding-sm); + box-shadow: var(--card-shadow); border-radius: var(--border-radius-lg); + padding: var(--padding-sm); } .block-menu-item-icon svg{ From 66c8fb9cfa578add6c6dcd0bf31aecfdaea6a32a Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Thu, 27 Jan 2022 12:24:22 +0530 Subject: [PATCH 04/68] fix: validate doc naming when set via prompt or by passing `set_name` --- frappe/model/document.py | 6 ++--- frappe/tests/test_naming.py | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index db4b7703ba..e482be0056 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -5,7 +5,7 @@ import time from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc +from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc, validate_name from werkzeug.exceptions import NotFound, Forbidden import hashlib, json from frappe.model import optional_fields, table_fields @@ -415,12 +415,12 @@ class Document(BaseDocument): # If autoname has set as Prompt (name) if self.get("__newname"): - self.name = self.get("__newname") + self.name = validate_name(self.doctype, self.get("__newname")) self.flags.name_set = True return if set_name: - self.name = set_name + self.name = validate_name(self.doctype, set_name) else: set_new_name(self) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 3031d3e344..c1aba4e88e 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -10,12 +10,18 @@ from frappe.model.naming import append_number_if_name_exists, revert_series_if_l from frappe.model.naming import determine_consecutive_week_number, parse_naming_series class TestNaming(unittest.TestCase): + def setUp(self): + frappe.db.sql('delete from `tabNote`') + frappe.db.sql('delete from `tabToDo`') + def tearDown(self): # Reset ToDo autoname to hash todo_doctype = frappe.get_doc('DocType', 'ToDo') todo_doctype.autoname = 'hash' todo_doctype.save() + frappe.db.rollback() + def test_append_number_if_name_exists(self): ''' Append number to name based on existing values @@ -208,3 +214,43 @@ class TestNaming(unittest.TestCase): dt = datetime.fromisoformat("2021-12-31") w = determine_consecutive_week_number(dt) self.assertEqual(w, "52") + + def test_naming_validations(self): + # case 1: check same name as doctype + # set name via prompt + tag = frappe.get_doc({ + 'doctype': 'Tag', + '__newname': 'Tag' + }) + self.assertRaises(frappe.NameError, tag.insert) + + # set by passing set_name as ToDo + self.assertRaises(frappe.NameError, make_invalid_todo) + + # set new name - Note + note = frappe.get_doc({ + 'doctype': 'Note', + 'title': 'Note' + }) + self.assertRaises(frappe.NameError, note.insert) + + # case 2: set name with "New ---" + tag = frappe.get_doc({ + 'doctype': 'Tag', + '__newname': 'New Tag' + }) + self.assertRaises(frappe.NameError, tag.insert) + + # case 3: set name with special characters + tag = frappe.get_doc({ + 'doctype': 'Tag', + '__newname': 'Tag<>' + }) + self.assertRaises(frappe.NameError, tag.insert) + + +def make_invalid_todo(): + frappe.get_doc({ + 'doctype': 'ToDo', + 'description': 'Test' + }).insert(set_name='ToDo') \ No newline at end of file From de4997b2d381e91e4a08c087656a8c398066a4ef Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 27 Jan 2022 14:42:38 +0530 Subject: [PATCH 05/68] fix: Close notification when any notification item is clicked --- .../public/js/frappe/ui/notifications/notifications.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/ui/notifications/notifications.js b/frappe/public/js/frappe/ui/notifications/notifications.js index 6fa8303574..35b1bed14e 100644 --- a/frappe/public/js/frappe/ui/notifications/notifications.js +++ b/frappe/public/js/frappe/ui/notifications/notifications.js @@ -283,12 +283,13 @@ class NotificationsView extends BaseNotificationsView { e.stopImmediatePropagation(); this.mark_as_read(field.name, item_html); }); - - item_html.on('click', () => { - this.mark_as_read(field.name, item_html); - }); } + item_html.on('click', () => { + !field.read && this.mark_as_read(field.name, item_html); + this.notifications_icon.trigger('click') + }); + return item_html; } From 8e21d18ee09bf74f2e2e48557ae5072632fb9349 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 27 Jan 2022 14:59:24 +0530 Subject: [PATCH 06/68] test: fixed workspace ui test --- cypress/integration/workspace.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cypress/integration/workspace.js b/cypress/integration/workspace.js index 9d6eeaff64..fbff451305 100644 --- a/cypress/integration/workspace.js +++ b/cypress/integration/workspace.js @@ -23,7 +23,7 @@ context('Workspace 2.0', () => { // check if sidebar item is added in pubic section cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0'); - cy.get('.standard-actions .btn-primary[data-label="Save Customizations"]').click(); + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); cy.wait(300); cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0'); @@ -67,7 +67,7 @@ context('Workspace 2.0', () => { cy.get('.ce-block:last .dropdown-item').contains('Expand').click(); cy.get(".ce-block:last").should('have.class', 'col-xs-12'); - cy.get('.standard-actions .btn-primary[data-label="Save Customizations"]').click(); + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); }); it('Delete Private Page', () => { @@ -80,7 +80,7 @@ context('Workspace 2.0', () => { .find('.dropdown-item[title="Delete Workspace"]').click({force: true}); cy.wait(300); cy.get('.modal-footer > .standard-actions > .btn-modal-primary:visible').first().click(); - cy.get('.standard-actions .btn-primary[data-label="Save Customizations"]').click(); + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); cy.get('.codex-editor__redactor .ce-block'); cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('not.exist'); }); From 2500d2c5873b98f5ce9acdb25b80d37990801b11 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 27 Jan 2022 15:00:57 +0530 Subject: [PATCH 07/68] fix: sider fix --- frappe/public/js/frappe/ui/notifications/notifications.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/notifications/notifications.js b/frappe/public/js/frappe/ui/notifications/notifications.js index 35b1bed14e..bf1cee2cbf 100644 --- a/frappe/public/js/frappe/ui/notifications/notifications.js +++ b/frappe/public/js/frappe/ui/notifications/notifications.js @@ -287,7 +287,7 @@ class NotificationsView extends BaseNotificationsView { item_html.on('click', () => { !field.read && this.mark_as_read(field.name, item_html); - this.notifications_icon.trigger('click') + this.notifications_icon.trigger('click'); }); return item_html; From b5ebf062cb9ca52f0f6f21f7dd37cf0f8d562f4f Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Thu, 27 Jan 2022 15:21:47 +0530 Subject: [PATCH 08/68] refactor: replace Link field with Select field --- frappe/desk/doctype/form_tour/form_tour.js | 111 ++++++++++-------- frappe/desk/doctype/form_tour/form_tour.py | 65 +++------- .../form_tour_step/form_tour_step.json | 49 ++------ frappe/public/js/frappe/form/form_tour.js | 2 +- 4 files changed, 90 insertions(+), 137 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 6a7c736fac..2e149c2c47 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -53,73 +53,84 @@ frappe.ui.form.on('Form Tour', { }; }); - frm.set_query("field", "steps", function() { - return { - query: "frappe.desk.doctype.form_tour.form_tour.get_docfield_list", - filters: { - doctype: frm.doc.reference_doctype, - hidden: 0 - } - }; - }); - - frm.set_query("parent_field", "steps", function() { - return { - query: "frappe.desk.doctype.form_tour.form_tour.get_docfield_list", - filters: { - doctype: frm.doc.reference_doctype, - fieldtype: "Table", - hidden: 0, - } - }; - }); - frm.trigger('reference_doctype'); }, reference_doctype(frm) { if (!frm.doc.reference_doctype) return; - frappe.db.get_list('DocField', { - filters: { - parent: frm.doc.reference_doctype, - parenttype: 'DocType', - fieldtype: 'Table' - }, - fields: ['options'] - }).then(res => { - if (Array.isArray(res)) { - frm.child_doctypes = res.map(r => r.options); - } + frappe.model.with_doctype(frm.doc.reference_doctype, () => { + let fields = frappe.meta + .get_docfields(frm.doc.reference_doctype, null, { + hidden: 0 + }) + .map(df => ({ + label: `${df.label || 'No Label'} (${df.fieldtype})`, + value: df.fieldname + })); + + frm.fields_dict.steps.grid.update_docfield_property( + "fieldname", + "options", + [""].concat(fields) + ); + + let parent_fields = frappe.meta + .get_docfields(frm.doc.reference_doctype, null, { + fieldtype: "Table", + hidden: 0 + }) + .map(df => ({ + label: `${df.label || 'No Label'} (${df.fieldtype})`, + value: df.fieldname + })); + + frm.fields_dict.steps.grid.update_docfield_property( + "parent_fieldname", + "options", + [""].concat(parent_fields) + ); }); } }); frappe.ui.form.on('Form Tour Step', { - parent_field(frm, cdt, cdn) { + form_render(frm, cdt, cdn) { + if (locals[cdt][cdn].is_table_field) { + frm.trigger('parent_fieldname', cdt, cdn); + } + }, + parent_fieldname(frm, cdt, cdn) { const child_row = locals[cdt][cdn]; - frappe.model.set_value(cdt, cdn, 'field', ''); - const field_control = get_child_field("steps", cdn, "field"); - field_control.get_query = function() { - return { - query: "frappe.desk.doctype.form_tour.form_tour.get_docfield_list", - filters: { - doctype: child_row.child_doctype, + + const parent_fieldname_df = frappe + .get_meta(frm.doc.reference_doctype) + .fields.find(df => df.fieldname == child_row.parent_fieldname); + + frappe.model.with_doctype(parent_fieldname_df.options, () => { + let fields = frappe.meta + .get_docfields(parent_fieldname_df.options, null, { hidden: 0 - } - }; - }; + }) + .map(df => ({ + label: `${df.label || 'No Label'} (${df.fieldtype})`, + value: df.fieldname + })); + + frm.fields_dict.steps.grid.update_docfield_property( + "fieldname", + "options", + [""].concat(fields) + ); + + if (child_row.fieldname) { + frappe.model.set_value(cdt, cdn, 'fieldname', child_row.fieldname); + } + }); } }); -function get_child_field(child_table, child_name, fieldname) { - // gets the field from grid row form - const grid = cur_frm.fields_dict[child_table].grid; - const grid_row = grid.grid_rows_by_docname[child_name]; - return grid_row.grid_form.fields_dict[fieldname]; -} - async function check_if_single(doctype) { const { message } = await frappe.db.get_value('DocType', doctype, 'issingle'); return message.issingle || 0; diff --git a/frappe/desk/doctype/form_tour/form_tour.py b/frappe/desk/doctype/form_tour/form_tour.py index 82d47224dd..6248b43e62 100644 --- a/frappe/desk/doctype/form_tour/form_tour.py +++ b/frappe/desk/doctype/form_tour/form_tour.py @@ -5,58 +5,23 @@ import frappe from frappe.model.document import Document from frappe.modules.export_file import export_to_files + class FormTour(Document): - def before_insert(self): - if not self.is_standard: - return + def before_save(self): + meta = frappe.get_meta(self.reference_doctype) + for step in self.steps: + if step.is_table_field and step.parent_fieldname: + parent_field_df = meta.get_field(step.parent_fieldname) + step.child_doctype = parent_field_df.options - # while syncing, set proper docfield reference - for d in self.steps: - if not frappe.db.exists('DocField', d.field): - d.field = frappe.db.get_value('DocField', { - 'fieldname': d.fieldname, 'parent': self.reference_doctype, 'fieldtype': d.fieldtype - }, "name") - - if d.is_table_field and not frappe.db.exists('DocField', d.parent_field): - d.parent_field = frappe.db.get_value('DocField', { - 'fieldname': d.parent_fieldname, 'parent': self.reference_doctype, 'fieldtype': 'Table' - }, "name") + field_df = frappe.get_meta(step.child_doctype).get_field(step.fieldname) + step.label = field_df.label + step.fieldtype = field_df.fieldtype + else: + field_df = meta.get_field(step.fieldname) + step.label = field_df.label + step.fieldtype = field_df.fieldtype def on_update(self): if frappe.conf.developer_mode and self.is_standard: - export_to_files([['Form Tour', self.name]], self.module) - - def before_export(self, doc): - for d in doc.steps: - d.field = "" - d.parent_field = "" - -@frappe.whitelist() -@frappe.validate_and_sanitize_search_inputs -def get_docfield_list(doctype, txt, searchfield, start, page_len, filters): - or_filters = [ - ['fieldname', 'like', '%' + txt + '%'], - ['label', 'like', '%' + txt + '%'], - ['fieldtype', 'like', '%' + txt + '%'] - ] - - parent_doctype = filters.get('doctype') - fieldtype = filters.get('fieldtype') - if not fieldtype: - excluded_fieldtypes = ['Column Break'] - excluded_fieldtypes += filters.get('excluded_fieldtypes', []) - fieldtype_filter = ['not in', excluded_fieldtypes] - else: - fieldtype_filter = fieldtype - - docfields = frappe.get_all( - doctype, - fields=["name as value", "label", "fieldtype"], - filters={'parent': parent_doctype, 'fieldtype': fieldtype_filter}, - or_filters=or_filters, - limit_start=start, - limit_page_length=page_len, - order_by="idx", - as_list=1, - ) - return docfields + export_to_files([["Form Tour", self.name]], self.module) diff --git a/frappe/desk/doctype/form_tour_step/form_tour_step.json b/frappe/desk/doctype/form_tour_step/form_tour_step.json index 3b6c91a208..7eb6eab223 100644 --- a/frappe/desk/doctype/form_tour_step/form_tour_step.json +++ b/frappe/desk/doctype/form_tour_step/form_tour_step.json @@ -6,19 +6,17 @@ "field_order": [ "is_table_field", "section_break_2", - "parent_field", - "field", + "parent_fieldname", + "fieldname", "title", "description", "column_break_2", "position", "label", + "fieldtype", "has_next_condition", "next_step_condition", "section_break_13", - "fieldname", - "parent_fieldname", - "fieldtype", "child_doctype" ], "fields": [ @@ -38,23 +36,13 @@ "reqd": 1 }, { - "depends_on": "eval: (!doc.is_table_field || (doc.is_table_field && doc.parent_field))", - "fieldname": "field", - "fieldtype": "Link", - "label": "Field", - "options": "DocField", + "depends_on": "eval: (!doc.is_table_field || (doc.is_table_field && doc.parent_fieldname))", + "fieldname": "fieldname", + "fieldtype": "Select", + "label": "Fieldname", "reqd": 1 }, { - "fetch_from": "field.fieldname", - "fieldname": "fieldname", - "fieldtype": "Data", - "hidden": 1, - "label": "Fieldname", - "read_only": 1 - }, - { - "fetch_from": "field.label", "fieldname": "label", "fieldtype": "Data", "in_list_view": 1, @@ -88,10 +76,8 @@ }, { "default": "0", - "fetch_from": "field.fieldtype", "fieldname": "fieldtype", "fieldtype": "Data", - "hidden": 1, "label": "Fieldtype", "read_only": 1 }, @@ -105,14 +91,6 @@ "fieldname": "section_break_2", "fieldtype": "Section Break" }, - { - "depends_on": "is_table_field", - "fieldname": "parent_field", - "fieldtype": "Link", - "label": "Parent Field", - "mandatory_depends_on": "is_table_field", - "options": "DocField" - }, { "fieldname": "section_break_13", "fieldtype": "Section Break", @@ -120,7 +98,6 @@ "label": "Hidden Fields" }, { - "fetch_from": "parent_field.options", "fieldname": "child_doctype", "fieldtype": "Data", "hidden": 1, @@ -128,18 +105,17 @@ "read_only": 1 }, { - "fetch_from": "parent_field.fieldname", + "depends_on": "is_table_field", "fieldname": "parent_fieldname", - "fieldtype": "Data", - "hidden": 1, - "label": "Parent Fieldname", - "read_only": 1 + "fieldtype": "Select", + "label": "Parent Field", + "mandatory_depends_on": "is_table_field" } ], "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2021-06-06 20:52:21.076972", + "modified": "2022-01-27 15:18:36.481801", "modified_by": "Administrator", "module": "Desk", "name": "Form Tour Step", @@ -147,5 +123,6 @@ "permissions": [], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 7fefb59ac6..2bb888e17c 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -151,7 +151,7 @@ frappe.ui.form.FormTour = class FormTour { const curr_step = step_info; const next_step = this.tour.steps[curr_step.idx]; - const is_next_field_in_curr_table = next_step.parent_field == curr_step.field; + const is_next_field_in_curr_table = next_step.parent_fieldname == curr_step.fieldname; if (!is_next_field_in_curr_table) return; From b3b1c3302c094ceaf0b1ff140a0d8ae3b45c3251 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 27 Jan 2022 19:18:39 +0530 Subject: [PATCH 09/68] chore: renamed tab break fields --- frappe/desk/doctype/workspace/workspace.json | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/frappe/desk/doctype/workspace/workspace.json b/frappe/desk/doctype/workspace/workspace.json index 960ae215e7..fa8b81f5fd 100644 --- a/frappe/desk/doctype/workspace/workspace.json +++ b/frappe/desk/doctype/workspace/workspace.json @@ -20,13 +20,13 @@ "hide_custom", "public", "content", - "section_break_2", + "tab_break_2", "charts", - "section_break_15", + "tab_break_15", "shortcuts", - "section_break_18", + "tab_break_18", "links", - "roles_section", + "roles_tab", "roles" ], "fields": [ @@ -40,7 +40,7 @@ { "collapsible": 1, "collapsible_depends_on": "charts", - "fieldname": "section_break_2", + "fieldname": "tab_break_2", "fieldtype": "Tab Break", "label": "Dashboards" }, @@ -78,14 +78,14 @@ { "collapsible": 1, "collapsible_depends_on": "shortcuts", - "fieldname": "section_break_15", + "fieldname": "tab_break_15", "fieldtype": "Tab Break", "label": "Shortcuts" }, { "collapsible": 1, "collapsible_depends_on": "links", - "fieldname": "section_break_18", + "fieldname": "tab_break_18", "fieldtype": "Tab Break", "label": "Link Cards" }, @@ -152,7 +152,7 @@ "options": "Has Role" }, { - "fieldname": "roles_section", + "fieldname": "roles_tab", "fieldtype": "Tab Break", "label": "Roles" } @@ -189,6 +189,5 @@ } ], "sort_field": "modified", - "sort_order": "DESC", - "states": [] + "sort_order": "DESC" } \ No newline at end of file From aba467307f8bc456960128360fe7acd75f5fa66d Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Thu, 27 Jan 2022 20:14:51 +0100 Subject: [PATCH 10/68] fix: translate default msgprint title in backend --- frappe/__init__.py | 5 ++--- frappe/public/js/frappe/ui/messages.js | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index c6cbfead43..f1e9987f17 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -356,7 +356,7 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False, response JSON and shown in a pop-up / modal. :param msg: Message. - :param title: [optional] Message title. + :param title: [optional] Message title. Default: "Message". :param raise_exception: [optional] Raise given exception and show message. :param as_table: [optional] If `msg` is a list of lists, render as HTML table. :param as_list: [optional] If `msg` is a list, render as un-ordered list. @@ -393,8 +393,7 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False, if flags.print_messages and out.message: print(f"Message: {strip_html_tags(out.message)}") - if title: - out.title = title + out.title = title or _("Message", context="Default title of the message dialog") if not indicator and raise_exception: indicator = 'red' diff --git a/frappe/public/js/frappe/ui/messages.js b/frappe/public/js/frappe/ui/messages.js index ac0c01c406..f0d03f0743 100644 --- a/frappe/public/js/frappe/ui/messages.js +++ b/frappe/public/js/frappe/ui/messages.js @@ -233,7 +233,7 @@ frappe.msgprint = function(msg, title, is_minimizable) { if(data.title || !msg_exists) { // set title only if it is explicitly given // and no existing title exists - frappe.msg_dialog.set_title(data.title || __('Message')); + frappe.msg_dialog.set_title(data.title || __('Message', null, 'Default title of the message dialog')); } // show / hide indicator From fd1006e619116a6d107397d595e85d62c86f46b1 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 28 Jan 2022 12:16:32 +0100 Subject: [PATCH 11/68] test: compare message, ignore title --- frappe/core/doctype/file/test_file.py | 2 +- frappe/core/doctype/user/test_user.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2c1042e104..6841f2fbd0 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -419,7 +419,7 @@ class TestFile(unittest.TestCase): test_file.reload() test_file.file_url = frappe.utils.get_url('unknown.jpg') test_file.make_thumbnail(suffix="xs") - self.assertEqual(json.loads(frappe.message_log[0]), {"message": f"File '{frappe.utils.get_url('unknown.jpg')}' not found"}) + self.assertEqual(json.loads(frappe.message_log[0]).get("message"), f"File '{frappe.utils.get_url('unknown.jpg')}' not found") self.assertEquals(test_file.thumbnail_url, None) def test_file_unzip(self): diff --git a/frappe/core/doctype/user/test_user.py b/frappe/core/doctype/user/test_user.py index d1291acfc4..581b44816a 100644 --- a/frappe/core/doctype/user/test_user.py +++ b/frappe/core/doctype/user/test_user.py @@ -355,7 +355,7 @@ class TestUser(unittest.TestCase): test_user.reload() self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/") update_password(old_password, old_password=new_password) - self.assertEqual(json.loads(frappe.message_log[0]), {"message": "Password reset instructions have been sent to your email"}) + self.assertEqual(json.loads(frappe.message_log[0]).get("message"), "Password reset instructions have been sent to your email") sendmail.assert_called_once() self.assertEqual(sendmail.call_args[1]["recipients"], "test2@example.com") From a5613720ed991d6bbab391592095e4742dcea422 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Fri, 28 Jan 2022 17:32:50 +0530 Subject: [PATCH 12/68] refactor: use set_fields_as_options --- frappe/desk/doctype/form_tour/form_tour.js | 53 ++++++++-------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 2e149c2c47..19fe623f60 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -59,36 +59,27 @@ frappe.ui.form.on('Form Tour', { reference_doctype(frm) { if (!frm.doc.reference_doctype) return; - frappe.model.with_doctype(frm.doc.reference_doctype, () => { - let fields = frappe.meta - .get_docfields(frm.doc.reference_doctype, null, { - hidden: 0 - }) - .map(df => ({ - label: `${df.label || 'No Label'} (${df.fieldtype})`, - value: df.fieldname - })); - + frm.set_fields_as_options( + "fieldname", + frm.doc.reference_doctype, + df => !df.hidden + ).then(options => { frm.fields_dict.steps.grid.update_docfield_property( "fieldname", "options", - [""].concat(fields) + [""].concat(options) ); + }); - let parent_fields = frappe.meta - .get_docfields(frm.doc.reference_doctype, null, { - fieldtype: "Table", - hidden: 0 - }) - .map(df => ({ - label: `${df.label || 'No Label'} (${df.fieldtype})`, - value: df.fieldname - })); - + frm.set_fields_as_options( + 'parent_fieldname', + frm.doc.reference_doctype, + (df) => df.fieldtype == "Table" && !df.hidden, + ).then(options => { frm.fields_dict.steps.grid.update_docfield_property( "parent_fieldname", "options", - [""].concat(parent_fields) + [""].concat(options) ); }); @@ -108,22 +99,16 @@ frappe.ui.form.on('Form Tour Step', { .get_meta(frm.doc.reference_doctype) .fields.find(df => df.fieldname == child_row.parent_fieldname); - frappe.model.with_doctype(parent_fieldname_df.options, () => { - let fields = frappe.meta - .get_docfields(parent_fieldname_df.options, null, { - hidden: 0 - }) - .map(df => ({ - label: `${df.label || 'No Label'} (${df.fieldtype})`, - value: df.fieldname - })); - + frm.set_fields_as_options( + 'fieldname', + parent_fieldname_df.options, + (df) => !df.hidden, + ).then(options => { frm.fields_dict.steps.grid.update_docfield_property( "fieldname", "options", - [""].concat(fields) + [""].concat(options) ); - if (child_row.fieldname) { frappe.model.set_value(cdt, cdn, 'fieldname', child_row.fieldname); } From 62643e835568d7799165409afa620b6d2c6519b2 Mon Sep 17 00:00:00 2001 From: shadrak gurupnor Date: Sat, 29 Jan 2022 11:45:51 +0530 Subject: [PATCH 13/68] fix: filter html tags in comments --- frappe/templates/includes/comments/comments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/templates/includes/comments/comments.py b/frappe/templates/includes/comments/comments.py index 99afb580d8..8d485423bf 100644 --- a/frappe/templates/includes/comments/comments.py +++ b/frappe/templates/includes/comments/comments.py @@ -6,6 +6,7 @@ from frappe.website.utils import clear_cache from frappe.rate_limiter import rate_limit from frappe.utils import add_to_date, now from frappe.website.doctype.blog_settings.blog_settings import get_comment_limit +from frappe.utils.html_utils import clean_html from frappe import _ @@ -29,7 +30,7 @@ def add_comment(comment, comment_email, comment_by, reference_doctype, reference return False comment = doc.add_comment( - text=comment, + text=clean_html(comment), comment_email=comment_email, comment_by=comment_by) From 105f8d8bc7868ae59922a895ad071d0927fadc9c Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Mon, 31 Jan 2022 11:24:08 +0530 Subject: [PATCH 14/68] fix: form tour ui test --- frappe/tests/ui_test_helpers.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index b299df522c..67c58a1154 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -148,9 +148,6 @@ def create_form_tour(): if frappe.db.exists('Form Tour', {'name': 'Test Form Tour'}): return - def get_docfield_name(filters): - return frappe.db.get_value('DocField', filters, "name") - tour = frappe.get_doc({ 'doctype': 'Form Tour', 'title': 'Test Form Tour', @@ -161,7 +158,6 @@ def create_form_tour(): "description": "Test Description 1", "has_next_condition": 1, "next_step_condition": "eval: doc.first_name", - "field": get_docfield_name({'parent': 'Contact', 'fieldname': 'first_name'}), "fieldname": "first_name", "fieldtype": "Data" },{ @@ -169,21 +165,18 @@ def create_form_tour(): "description": "Test Description 2", "has_next_condition": 1, "next_step_condition": "eval: doc.last_name", - "field": get_docfield_name({'parent': 'Contact', 'fieldname': 'last_name'}), "fieldname": "last_name", "fieldtype": "Data" },{ "title": "Test Title 3", "description": "Test Description 3", - "field": get_docfield_name({'parent': 'Contact', 'fieldname': 'phone_nos'}), "fieldname": "phone_nos", "fieldtype": "Table" },{ "title": "Test Title 4", "description": "Test Description 4", "is_table_field": 1, - "parent_field": get_docfield_name({'parent': 'Contact', 'fieldname': 'phone_nos'}), - "field": get_docfield_name({'parent': 'Contact Phone', 'fieldname': 'phone'}), + "parent_fieldname": "phone_nos", "next_step_condition": "eval: doc.phone", "has_next_condition": 1, "fieldname": "phone", From 62331fcba3acbdc6c00d15cb6ee1b85a85dccf1b Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Mon, 31 Jan 2022 13:49:37 +0530 Subject: [PATCH 15/68] test: validation for no name specified --- frappe/tests/test_naming.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index c1aba4e88e..16bfeb4858 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -248,6 +248,13 @@ class TestNaming(unittest.TestCase): }) self.assertRaises(frappe.NameError, tag.insert) + # case 4: no name specified + tag = frappe.get_doc({ + 'doctype': 'Tag', + '__newname': '' + }) + self.assertRaises(frappe.ValidationError, tag.insert) + def make_invalid_todo(): frappe.get_doc({ From 1a1d0849d0857e21777a032aabdb1ddbeedd6dd0 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 31 Jan 2022 15:03:07 +0530 Subject: [PATCH 16/68] fix: check if any changes made before saving --- frappe/public/js/frappe/views/workspace/workspace.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index 2a02cd33ce..5e1a659cbd 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -1158,7 +1158,7 @@ frappe.views.Workspace = class Workspace { item.data.card_name !== 'Custom Reports') ); - if (page.content == JSON.stringify(blocks)) { + if (page.content == JSON.stringify(blocks) && Object.keys(new_widgets).length === 0) { this.setup_customization_buttons(page); frappe.show_alert({ message: __("No changes made on the page"), indicator: "warning" }); return false; From 20f35b8b6c4897c7201605c62f761b34b1634168 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 31 Jan 2022 15:04:17 +0530 Subject: [PATCH 17/68] fix: Make card links mandatory --- .../public/js/frappe/widgets/widget_dialog.js | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index 01d41a0cf9..1486f94945 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -154,7 +154,7 @@ class CardDialog extends WidgetDialog { { fieldtype: "Data", fieldname: "label", - label: "Label", + label: "Label" }, { fieldname: 'links', @@ -174,7 +174,7 @@ class CardDialog extends WidgetDialog { }, { fieldname: "icon", - fieldtype: "Data", + fieldtype: "Icon", label: "Icon" }, { @@ -182,6 +182,7 @@ class CardDialog extends WidgetDialog { fieldtype: "Select", in_list_view: 1, label: "Link Type", + reqd: 1, options: ["DocType", "Page", "Report"] }, { @@ -189,9 +190,9 @@ class CardDialog extends WidgetDialog { fieldtype: "Dynamic Link", in_list_view: 1, label: "Link To", + reqd: 1, get_options: (df) => { return df.doc.link_type; - } }, { @@ -227,6 +228,31 @@ class CardDialog extends WidgetDialog { } process_data(data) { + data.links.map((item, idx) => { + let message = ''; + let row = idx+1; + + if (!item.link_type) { + message = "Following fields have missing values:

    " + message += `
  • Link Type in Row ${row}
    • `; + } + + if (!item.link_to) { + message += `
  • Link To in Row ${row}
    • `; + } + + if (message) { + message += "
"; + frappe.throw({ + message:__(message), + title: __("Missing Values Required"), + indicator: 'orange' + }); + } + + item.label = item.label ? item.label : item.link_to; + }); + data.label = data.label ? data.label : data.chart_name; return data; } From c30e59eb8d96a4ab70ed453d236b679b30a2bdbc Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 31 Jan 2022 15:09:05 +0530 Subject: [PATCH 18/68] fix: sider fix --- frappe/public/js/frappe/widgets/widget_dialog.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index 1486f94945..d1ba75227b 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -233,7 +233,7 @@ class CardDialog extends WidgetDialog { let row = idx+1; if (!item.link_type) { - message = "Following fields have missing values:

    " + message = "Following fields have missing values:

      "; message += `
    • Link Type in Row ${row}
      • `; } @@ -244,7 +244,7 @@ class CardDialog extends WidgetDialog { if (message) { message += "
    "; frappe.throw({ - message:__(message), + message: __(message), title: __("Missing Values Required"), indicator: 'orange' }); From 7829de986204b8b6dfe35003e2fdf771347304e8 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 1 Feb 2022 06:22:39 +0530 Subject: [PATCH 19/68] fix: auto creation of append to doctype ref while receiving mail --- .../doctype/email_account/email_account.py | 7 +++-- frappe/email/receive.py | 30 ++++++++++++------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index a05e20da24..7374fe861a 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -463,11 +463,11 @@ class EmailAccount(Document): """ mails = [] - def process_mail(messages): + def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages['uid_list'][index] if messages.get('uid_list') else None seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0 - mails.append(InboundMail(message, self, uid, seen_status)) + mails.append(InboundMail(message, self, uid, append_to, seen_status)) if frappe.local.flags.in_test: return [InboundMail(msg, self) for msg in test_mails or []] @@ -484,7 +484,8 @@ class EmailAccount(Document): email_server.select_imap_folder(folder.folder_name) email_server.settings['uid_validity'] = folder.uidvalidity messages = email_server.get_messages(folder=folder.folder_name) or {} - process_mail(messages) + append_to = folder.append_to + process_mail(messages, append_to) else: # process the pop3 account messages = email_server.get_messages() or {} diff --git a/frappe/email/receive.py b/frappe/email/receive.py index dd64d0df80..16119ae25e 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -582,10 +582,11 @@ class Email: class InboundMail(Email): """Class representation of incoming mail along with mail handlers. """ - def __init__(self, content, email_account, uid=None, seen_status=None): + def __init__(self, content, email_account, uid=None, append_to=None, seen_status=None): super().__init__(content) self.email_account = email_account self.uid = uid or -1 + self.append_to = append_to or -1 self.seen_status = seen_status or 0 # System documents related to this mail @@ -623,15 +624,24 @@ class InboundMail(Email): if self.parent_communication(): data['in_reply_to'] = self.parent_communication().name - if self.reference_document(): - data['reference_doctype'] = self.reference_document().doctype - data['reference_name'] = self.reference_document().name - elif self.email_account.append_to and self.email_account.append_to != 'Communication': - reference_doc = self._create_reference_document(self.email_account.append_to) - if reference_doc: - data['reference_doctype'] = reference_doc.doctype - data['reference_name'] = reference_doc.name - data['is_first'] = True + if self.email_account.use_imap and self.append_to: + if self.append_to != 'Communication': + reference_doc = self._create_reference_document(self.append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True + else: + if self.reference_document(): + data['reference_doctype'] = self.reference_document().doctype + data['reference_name'] = self.reference_document().name + elif self.email_account.append_to and self.email_account.append_to != 'Communication': + reference_doc = self._create_reference_document(self.email_account.append_to) + # TODO: here instead of using email_account.append_to, the imap_folder.append_to should be used + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True if self.is_notification(): # Disable notifications for notification. From 988771a8c3dc9ed847d1bca192b99f64518be208 Mon Sep 17 00:00:00 2001 From: shadrak gurupnor Date: Tue, 1 Feb 2022 10:32:27 +0530 Subject: [PATCH 20/68] test(blog comment): added filter for blog comments --- frappe/core/doctype/comment/test_comment.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/core/doctype/comment/test_comment.py b/frappe/core/doctype/comment/test_comment.py index cd9af498aa..33672a7dea 100644 --- a/frappe/core/doctype/comment/test_comment.py +++ b/frappe/core/doctype/comment/test_comment.py @@ -70,6 +70,19 @@ class TestComment(unittest.TestCase): reference_name = test_blog.name ))), 0) + # test for filtering html and css injection elements + frappe.db.delete("Comment", {"reference_doctype": "Blog Post"}) + + frappe.form_dict.comment = 'Comment' + frappe.form_dict.comment_by = 'hacker' + + add_comment() + + self.assertEqual(frappe.get_all('Comment', fields = ['content'], filters = dict( + reference_doctype = test_blog.doctype, + reference_name = test_blog.name + ))[0]['content'], 'Comment') + test_blog.delete() From 57a6ee392a0fc94ac54188ffbf0f5943c7c859b0 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 1 Feb 2022 11:00:50 +0530 Subject: [PATCH 21/68] fix: append_to parameter order in InboundMail --- frappe/email/doctype/email_account/email_account.py | 2 +- frappe/email/receive.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 7374fe861a..ee230d33f0 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -467,7 +467,7 @@ class EmailAccount(Document): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages['uid_list'][index] if messages.get('uid_list') else None seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0 - mails.append(InboundMail(message, self, uid, append_to, seen_status)) + mails.append(InboundMail(message, self, uid, seen_status, append_to)) if frappe.local.flags.in_test: return [InboundMail(msg, self) for msg in test_mails or []] diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 16119ae25e..cc4ec23e4e 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -582,7 +582,7 @@ class Email: class InboundMail(Email): """Class representation of incoming mail along with mail handlers. """ - def __init__(self, content, email_account, uid=None, append_to=None, seen_status=None): + def __init__(self, content, email_account, uid=None, seen_status=None, append_to=None): super().__init__(content) self.email_account = email_account self.uid = uid or -1 From eb574ca9f1f39c54d96105a94e0e8a3a4a12964b Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 1 Feb 2022 19:01:41 +0530 Subject: [PATCH 22/68] feat: add filter for append_to wrt imap_folder --- frappe/email/doctype/email_account/email_account.py | 10 +++++----- frappe/email/receive.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index ee230d33f0..772173fcd2 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -481,11 +481,11 @@ class EmailAccount(Document): if self.use_imap: # process all given imap folder for folder in self.imap_folder: - email_server.select_imap_folder(folder.folder_name) - email_server.settings['uid_validity'] = folder.uidvalidity - messages = email_server.get_messages(folder=folder.folder_name) or {} - append_to = folder.append_to - process_mail(messages, append_to) + if email_server.select_imap_folder(folder.folder_name): + frappe.log_error(f'FOLDER NAME: {folder.folder_name} PRESENT') + email_server.settings['uid_validity'] = folder.uidvalidity + messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {} + process_mail(messages, folder.append_to) else: # process the pop3 account messages = email_server.get_messages() or {} diff --git a/frappe/email/receive.py b/frappe/email/receive.py index cc4ec23e4e..30dd146fa7 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -108,7 +108,8 @@ class EmailServer: raise def select_imap_folder(self, folder): - self.imap.select(folder) + res = self.imap.select(f'"{folder}"') + return res[0] == 'OK' # The folder exsits TODO: handle other resoponses too def logout(self): if cint(self.settings.use_imap): From 43ac81be5ddd00a4352a089d0571a1075595b521 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 1 Feb 2022 19:04:18 +0530 Subject: [PATCH 23/68] refactor: remove debug logs --- frappe/email/doctype/email_account/email_account.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 772173fcd2..1c53ae9faf 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -482,7 +482,6 @@ class EmailAccount(Document): # process all given imap folder for folder in self.imap_folder: if email_server.select_imap_folder(folder.folder_name): - frappe.log_error(f'FOLDER NAME: {folder.folder_name} PRESENT') email_server.settings['uid_validity'] = folder.uidvalidity messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {} process_mail(messages, folder.append_to) From 58db3765098ad6ea9f6572ebc0fc286eb9b05251 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 1 Feb 2022 20:06:26 +0530 Subject: [PATCH 24/68] fix: email syncing with "seen" emails even if email_sync filter set to "unseen" issue --- frappe/email/doctype/email_account/email_account.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 1c53ae9faf..c3ab11d2e9 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -467,7 +467,9 @@ class EmailAccount(Document): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages['uid_list'][index] if messages.get('uid_list') else None seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0 - mails.append(InboundMail(message, self, uid, seen_status, append_to)) + if not (self.email_sync_option == 'UNSEEN' and seen_status): + # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' + mails.append(InboundMail(message, self, uid, seen_status, append_to)) if frappe.local.flags.in_test: return [InboundMail(msg, self) for msg in test_mails or []] From 18820bf44e56cbfee380bcc2b037f46ea87ec487 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Wed, 2 Feb 2022 09:08:03 +0530 Subject: [PATCH 25/68] feat: add test --- .../email/doctype/email_account/test_email_account.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 6d26f9f070..d59de050c4 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -246,6 +246,16 @@ class TestEmailAccount(unittest.TestCase): with self.assertRaises(Exception): email_account.validate() + def test_append_to(self): + email_aacount = frappe.get_doc("Email Account", "_Test Email Account 1") + mail_content = self.get_test_mail(fname="incoming-2.raw") + + inbound_mail = InboundMail(mail_content, email_aacount, 12345, 1, 'ToDo') + communication = inbound_mail.process() + if inbound_mail.append_to == 'ToDo': + self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertTrue(communication.reference_name) + class TestInboundMail(unittest.TestCase): @classmethod def setUpClass(cls): From f960fb70dcc49f6a9ceab7edeeab8b66dc04aa44 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Wed, 2 Feb 2022 10:14:46 +0530 Subject: [PATCH 26/68] fix: append_to in case of not passed issue --- frappe/email/receive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 30dd146fa7..7b006c32b0 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -587,7 +587,7 @@ class InboundMail(Email): super().__init__(content) self.email_account = email_account self.uid = uid or -1 - self.append_to = append_to or -1 + self.append_to = append_to self.seen_status = seen_status or 0 # System documents related to this mail @@ -625,8 +625,8 @@ class InboundMail(Email): if self.parent_communication(): data['in_reply_to'] = self.parent_communication().name - if self.email_account.use_imap and self.append_to: - if self.append_to != 'Communication': + if self.email_account.use_imap: + if self.append_to and self.append_to != 'Communication': reference_doc = self._create_reference_document(self.append_to) if reference_doc: data['reference_doctype'] = reference_doc.doctype From 12442726c6e6627523d2ad1430fb5cc4c72f8efc Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Wed, 2 Feb 2022 11:00:39 +0530 Subject: [PATCH 27/68] fix: reply emails get_reference_doc from communication issue --- frappe/email/receive.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 7b006c32b0..8bbc4611ff 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -624,25 +624,18 @@ class InboundMail(Email): if self.parent_communication(): data['in_reply_to'] = self.parent_communication().name + + append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to - if self.email_account.use_imap: - if self.append_to and self.append_to != 'Communication': - reference_doc = self._create_reference_document(self.append_to) - if reference_doc: - data['reference_doctype'] = reference_doc.doctype - data['reference_name'] = reference_doc.name - data['is_first'] = True - else: - if self.reference_document(): - data['reference_doctype'] = self.reference_document().doctype - data['reference_name'] = self.reference_document().name - elif self.email_account.append_to and self.email_account.append_to != 'Communication': - reference_doc = self._create_reference_document(self.email_account.append_to) - # TODO: here instead of using email_account.append_to, the imap_folder.append_to should be used - if reference_doc: - data['reference_doctype'] = reference_doc.doctype - data['reference_name'] = reference_doc.name - data['is_first'] = True + if self.reference_document(): + data['reference_doctype'] = self.reference_document().doctype + data['reference_name'] = self.reference_document().name + elif append_to and append_to != 'Communication': + reference_doc = self._create_reference_document(append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True if self.is_notification(): # Disable notifications for notification. From ffbf215070b07439f80e78a70a817ef69066c48b Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Wed, 2 Feb 2022 11:29:43 +0530 Subject: [PATCH 28/68] fix: communication if is_first and no append_to issue --- frappe/email/receive.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 8bbc4611ff..b8156d5d9b 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -624,18 +624,19 @@ class InboundMail(Email): if self.parent_communication(): data['in_reply_to'] = self.parent_communication().name - + append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to if self.reference_document(): data['reference_doctype'] = self.reference_document().doctype data['reference_name'] = self.reference_document().name - elif append_to and append_to != 'Communication': - reference_doc = self._create_reference_document(append_to) - if reference_doc: - data['reference_doctype'] = reference_doc.doctype - data['reference_name'] = reference_doc.name - data['is_first'] = True + else: + if append_to and append_to != 'Communication': + reference_doc = self._create_reference_document(append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True if self.is_notification(): # Disable notifications for notification. From 3734dc792711097e5a7e4077f68e5c4c7f3f6edc Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Fri, 4 Feb 2022 15:07:18 +0530 Subject: [PATCH 29/68] feat: add tests for imap folders email syncing with multiple folders --- .../doctype/email_account/email_account.py | 10 ++- .../email_account/test_email_account.py | 60 ++++++++++++-- .../email_account/test_mails/messages-1.raw | 81 +++++++++++++++++++ .../doctype/email_account/test_records.json | 2 +- 4 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 frappe/email/doctype/email_account/test_mails/messages-1.raw diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index c3ab11d2e9..6d84cbb270 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -457,7 +457,7 @@ class EmailAccount(Document): if exceptions: raise Exception(frappe.as_json(exceptions)) - def get_inbound_mails(self, test_mails=None) -> List[InboundMail]: + def get_inbound_mails(self, test_mails=None, messages=None) -> List[InboundMail]: """retrive and return inbound mails. """ @@ -472,7 +472,13 @@ class EmailAccount(Document): mails.append(InboundMail(message, self, uid, seen_status, append_to)) if frappe.local.flags.in_test: - return [InboundMail(msg, self) for msg in test_mails or []] + if self.enable_incoming: + for folder in self.imap_folder: + _messages = messages[folder.folder_name] if folder.folder_name in messages and ( messages[folder.folder_name] is not None) else {} + process_mail(_messages, folder.append_to) + return mails + else: + return [InboundMail(msg, self) for msg in test_mails or []] if not self.enable_incoming: return [] diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index d59de050c4..3555353259 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -247,14 +247,64 @@ class TestEmailAccount(unittest.TestCase): email_account.validate() def test_append_to(self): - email_aacount = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") mail_content = self.get_test_mail(fname="incoming-2.raw") - inbound_mail = InboundMail(mail_content, email_aacount, 12345, 1, 'ToDo') + inbound_mail = InboundMail(mail_content, email_account, 12345, 1, 'ToDo') communication = inbound_mail.process() - if inbound_mail.append_to == 'ToDo': - self.assertEqual(communication.reference_doctype, 'ToDo') - self.assertTrue(communication.reference_name) + # the append_to for the email is set to ToDO in "_Test Email Account 1" + self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertTrue(communication.reference_name) + self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) + + def test_append_to_with_imap_folders(self): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + + mail_content_1 = self.get_test_mail(fname="incoming-1.raw") + mail_content_2 = self.get_test_mail(fname="incoming-2.raw") + mail_content_3 = self.get_test_mail(fname="incoming-3.raw") + + messages = { + 'INBOX': { # append_to = ToDo + 'latest_messages': [ + mail_content_1, + mail_content_2 + ], + 'seen_status': { + 0: 'UNSEEN', + 1: 'UNSEEN' + }, + 'uid_list': [0,1] + }, + 'Test Folder': { # append_to = Communication + 'latest_messages': [ + mail_content_3 + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + mails = email_account.get_inbound_mails(messages=messages) + self.assertEqual(len(mails), 3) + + inbox_mails = 0 + test_folder_mails = 0 + + for mail in mails: + communication = mail.process() + if mail.append_to == 'ToDo': + inbox_mails += 1 + self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertTrue(communication.reference_name) + self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) + else: + test_folder_mails += 1 + self.assertEqual(communication.reference_doctype, None) + + self.assertEqual(inbox_mails, 2) + self.assertEqual(test_folder_mails, 1) class TestInboundMail(unittest.TestCase): @classmethod diff --git a/frappe/email/doctype/email_account/test_mails/messages-1.raw b/frappe/email/doctype/email_account/test_mails/messages-1.raw new file mode 100644 index 0000000000..d0ac914371 --- /dev/null +++ b/frappe/email/doctype/email_account/test_mails/messages-1.raw @@ -0,0 +1,81 @@ +Delivered-To: kamal@frappe.io +Received: by 2002:a59:bcaf:0:b0:283:bf56:ecc6 with SMTP id q15csp1608707vqy; + Tue, 1 Feb 2022 18:36:19 -0800 (PST) +X-Received: by 2002:a05:6638:d06:: with SMTP id q6mr5074243jaj.197.1643769379814; + Tue, 01 Feb 2022 18:36:19 -0800 (PST) +ARC-Seal: i=1; a=rsa-sha256; t=1643769379; cv=none; + d=google.com; s=arc-20160816; + b=V0+D/66dKKmy7d5BdoHKohecWHF6PhMgtgarnemPm8UaOL5S5WyGUWt+azZZLK0HcU + TkxA3HY+/UlO3BjdmvY9hHtprPzTgK0U27S7sboIEQJXLblKfx9OOL2Wq0cS6YSlTcXQ + OIZaDepuJ9u7KUhRiHbmQCYXWmDdG0syALsONwjfXVBwqHn9wGizHbkp7tBCrgGbSfiF + NQgNOiwnau1jC48T+IKm7+9/1gqSWpU5c+4iWEDzmDaUTyE69XrHoAsThPXQ1FTn7beC + taVnDkMXIpJcOBJZLIWJLCKhju2oBz5EkX4j/IkNAnOIw40RAV/s9HT5DZUDjPoMd8g+ + IrGw== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; + h=to:subject:message-id:date:from:mime-version:dkim-signature; + bh=8ECAAGQ3yaTI5Xt9rqRpk+teGEit3tpBNmAS2780QKw=; + b=OjYxw0kvdSTP3Z+jAiU8LQJ65Ej6FAdZxEpUb8xvpFiP96kvRKkGEGhMFabPXDUXMn + mEeKI1Ocqm55b1XUWY44EK5E/eecA651SbIKvR/ihOXzrnPD3hItfj9S2PsgjyQp+rX1 + +vREp5o1A+tWbFI3smRYPqWUv1g7J5hRzCXjI1FyNI1OdkJn7K9PlEsR1xqjflGDmOif + nfglNmrkjCMIw9XHQ91rMY1ta+yrVvVeDFVLLBV4R0yCL/DrpmozSDViZPNzMmo6ITRQ + T+/qkuku9aBReM1XebA7mLKPrPGuU27l2GJX4hpsz0gEYDpItf53MEaO5XIkoELo0b8s + AxRQ== +ARC-Authentication-Results: i=1; mx.google.com; + dkim=pass header.i=@gmail.com header.s=20210112 header.b=J6sQ2r3s; + spf=pass (google.com: domain of kamaljohnson12345@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=kamaljohnson12345@gmail.com; + dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com +Return-Path: +Received: from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41]) + by mx.google.com with SMTPS id 1sor6796918jao.124.2022.02.01.18.36.19 + for + (Google Transport Security); + Tue, 01 Feb 2022 18:36:19 -0800 (PST) +Received-SPF: pass (google.com: domain of kamaljohnson12345@gmail.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41; +Authentication-Results: mx.google.com; + dkim=pass header.i=@gmail.com header.s=20210112 header.b=J6sQ2r3s; + spf=pass (google.com: domain of kamaljohnson12345@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=kamaljohnson12345@gmail.com; + dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=gmail.com; s=20210112; + h=mime-version:from:date:message-id:subject:to; + bh=8ECAAGQ3yaTI5Xt9rqRpk+teGEit3tpBNmAS2780QKw=; + b=J6sQ2r3s24Gj5ATpKCyjleWOaF9+9rvwKg/NJ979EpEvy13v4rRX4gL7xbX8GIVkZ4 + QoOl3xUNtRItJPYWuW984GXsmL5v8YrakCb0JzS5iWk3VC+Q+orhk95giHDLx/1UrTeK + KFhLQ632i23M2IutOA4vRVet3n4wnmYwXiJZ1G/HoE70h+1Jn+E0nsvBsz8IU5eWHve7 + SKWAaEtUOwcyuLi9SncQh62bM9ZW6V034LEb0MXz/suAJ+sPIvjSSNJwehKpvYfg1AUt + CkzuAxJt17MThTwz+/YCZkttN7LKDeBAUJWh2+JB+BfyvzYR1pUyJfwLhRMNYoeWZpGK + L/6g== +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20210112; + h=x-gm-message-state:mime-version:from:date:message-id:subject:to; + bh=8ECAAGQ3yaTI5Xt9rqRpk+teGEit3tpBNmAS2780QKw=; + b=CyVqLYduXIsBE7f62ti9wFFkTv/Vn6anf1UtnakmhgeWksW0nvjOjnAdytXHiD5Phl + vtTpmCGgLVcTZBI8wUaKdjsGDVWnpG8zYRQljytyrptyi+M4ShnUMq2WbyceeFFop+Ta + 5z/0PE3wva+H45uj2ICRu2l0o1nvjNKxt14zzbaCYgUyx2ZF24MLgIO2ko1xnTXS6M5g + ASnc4ia+9r3MhLa3y3Ss97KYRcLXeVzbqEXS4X3DB1h+/tHLogLTAX5D8w5LME7M0MfR + uy4dC0K4VmhhDJTzoFHzgBqWw1vqNU/G92Fr/kZdVzMiSegt0Up76ePA8mwpeEKzaNdP + e4wg== +X-Gm-Message-State: AOAM532tem+DamkNGjOKo5Ayzg9Gdh0clTzSw+ajeLwzgcNy/fNYBVhC + YHvVK8Ylvd0Jr8XjwshuyN/hz8UOoV2fkjOij9CED3Vo +X-Google-Smtp-Source: ABdhPJwB/h4hp6eiaPL7IIkokMwriJ92sqjBOp3ptJnp+mtkZNAr8JNrt86gahMU24MuTsB8HUnK9oEXGRKV+55XO5U= +X-Received: by 2002:a02:c916:: with SMTP id t22mr15608489jao.300.1643769379473; + Tue, 01 Feb 2022 18:36:19 -0800 (PST) +MIME-Version: 1.0 +From: Kamal Johnson +Date: Wed, 2 Feb 2022 08:06:06 +0530 +Message-ID: +Subject: Hfruuhhs +To: Kamal Johnson +Content-Type: multipart/alternative; boundary="000000000000f4cb7505d6ffe106" + +--000000000000f4cb7505d6ffe106 +Content-Type: text/plain; charset="UTF-8" + +Yysjtshsg + +--000000000000f4cb7505d6ffe106 +Content-Type: text/html; charset="UTF-8" + +
    Yysjtshsg
    + +--000000000000f4cb7505d6ffe106-- \ No newline at end of file diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json index 450895d7a6..66eb5a9b2e 100644 --- a/frappe/email/doctype/email_account/test_records.json +++ b/frappe/email/doctype/email_account/test_records.json @@ -20,7 +20,7 @@ "pop3_server": "pop.test.example.com", "no_remaining":"0", "append_to": "ToDo", - "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], + "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}], "track_email_status": 1 }, { From 8cc26af272c103d3f984707e11d635d95f7430a6 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Fri, 4 Feb 2022 15:28:16 +0530 Subject: [PATCH 30/68] fix: remove messages-1.raw --- .../email_account/test_mails/messages-1.raw | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 frappe/email/doctype/email_account/test_mails/messages-1.raw diff --git a/frappe/email/doctype/email_account/test_mails/messages-1.raw b/frappe/email/doctype/email_account/test_mails/messages-1.raw deleted file mode 100644 index d0ac914371..0000000000 --- a/frappe/email/doctype/email_account/test_mails/messages-1.raw +++ /dev/null @@ -1,81 +0,0 @@ -Delivered-To: kamal@frappe.io -Received: by 2002:a59:bcaf:0:b0:283:bf56:ecc6 with SMTP id q15csp1608707vqy; - Tue, 1 Feb 2022 18:36:19 -0800 (PST) -X-Received: by 2002:a05:6638:d06:: with SMTP id q6mr5074243jaj.197.1643769379814; - Tue, 01 Feb 2022 18:36:19 -0800 (PST) -ARC-Seal: i=1; a=rsa-sha256; t=1643769379; cv=none; - d=google.com; s=arc-20160816; - b=V0+D/66dKKmy7d5BdoHKohecWHF6PhMgtgarnemPm8UaOL5S5WyGUWt+azZZLK0HcU - TkxA3HY+/UlO3BjdmvY9hHtprPzTgK0U27S7sboIEQJXLblKfx9OOL2Wq0cS6YSlTcXQ - OIZaDepuJ9u7KUhRiHbmQCYXWmDdG0syALsONwjfXVBwqHn9wGizHbkp7tBCrgGbSfiF - NQgNOiwnau1jC48T+IKm7+9/1gqSWpU5c+4iWEDzmDaUTyE69XrHoAsThPXQ1FTn7beC - taVnDkMXIpJcOBJZLIWJLCKhju2oBz5EkX4j/IkNAnOIw40RAV/s9HT5DZUDjPoMd8g+ - IrGw== -ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; - h=to:subject:message-id:date:from:mime-version:dkim-signature; - bh=8ECAAGQ3yaTI5Xt9rqRpk+teGEit3tpBNmAS2780QKw=; - b=OjYxw0kvdSTP3Z+jAiU8LQJ65Ej6FAdZxEpUb8xvpFiP96kvRKkGEGhMFabPXDUXMn - mEeKI1Ocqm55b1XUWY44EK5E/eecA651SbIKvR/ihOXzrnPD3hItfj9S2PsgjyQp+rX1 - +vREp5o1A+tWbFI3smRYPqWUv1g7J5hRzCXjI1FyNI1OdkJn7K9PlEsR1xqjflGDmOif - nfglNmrkjCMIw9XHQ91rMY1ta+yrVvVeDFVLLBV4R0yCL/DrpmozSDViZPNzMmo6ITRQ - T+/qkuku9aBReM1XebA7mLKPrPGuU27l2GJX4hpsz0gEYDpItf53MEaO5XIkoELo0b8s - AxRQ== -ARC-Authentication-Results: i=1; mx.google.com; - dkim=pass header.i=@gmail.com header.s=20210112 header.b=J6sQ2r3s; - spf=pass (google.com: domain of kamaljohnson12345@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=kamaljohnson12345@gmail.com; - dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com -Return-Path: -Received: from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41]) - by mx.google.com with SMTPS id 1sor6796918jao.124.2022.02.01.18.36.19 - for - (Google Transport Security); - Tue, 01 Feb 2022 18:36:19 -0800 (PST) -Received-SPF: pass (google.com: domain of kamaljohnson12345@gmail.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41; -Authentication-Results: mx.google.com; - dkim=pass header.i=@gmail.com header.s=20210112 header.b=J6sQ2r3s; - spf=pass (google.com: domain of kamaljohnson12345@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=kamaljohnson12345@gmail.com; - dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com -DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; - d=gmail.com; s=20210112; - h=mime-version:from:date:message-id:subject:to; - bh=8ECAAGQ3yaTI5Xt9rqRpk+teGEit3tpBNmAS2780QKw=; - b=J6sQ2r3s24Gj5ATpKCyjleWOaF9+9rvwKg/NJ979EpEvy13v4rRX4gL7xbX8GIVkZ4 - QoOl3xUNtRItJPYWuW984GXsmL5v8YrakCb0JzS5iWk3VC+Q+orhk95giHDLx/1UrTeK - KFhLQ632i23M2IutOA4vRVet3n4wnmYwXiJZ1G/HoE70h+1Jn+E0nsvBsz8IU5eWHve7 - SKWAaEtUOwcyuLi9SncQh62bM9ZW6V034LEb0MXz/suAJ+sPIvjSSNJwehKpvYfg1AUt - CkzuAxJt17MThTwz+/YCZkttN7LKDeBAUJWh2+JB+BfyvzYR1pUyJfwLhRMNYoeWZpGK - L/6g== -X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; - d=1e100.net; s=20210112; - h=x-gm-message-state:mime-version:from:date:message-id:subject:to; - bh=8ECAAGQ3yaTI5Xt9rqRpk+teGEit3tpBNmAS2780QKw=; - b=CyVqLYduXIsBE7f62ti9wFFkTv/Vn6anf1UtnakmhgeWksW0nvjOjnAdytXHiD5Phl - vtTpmCGgLVcTZBI8wUaKdjsGDVWnpG8zYRQljytyrptyi+M4ShnUMq2WbyceeFFop+Ta - 5z/0PE3wva+H45uj2ICRu2l0o1nvjNKxt14zzbaCYgUyx2ZF24MLgIO2ko1xnTXS6M5g - ASnc4ia+9r3MhLa3y3Ss97KYRcLXeVzbqEXS4X3DB1h+/tHLogLTAX5D8w5LME7M0MfR - uy4dC0K4VmhhDJTzoFHzgBqWw1vqNU/G92Fr/kZdVzMiSegt0Up76ePA8mwpeEKzaNdP - e4wg== -X-Gm-Message-State: AOAM532tem+DamkNGjOKo5Ayzg9Gdh0clTzSw+ajeLwzgcNy/fNYBVhC - YHvVK8Ylvd0Jr8XjwshuyN/hz8UOoV2fkjOij9CED3Vo -X-Google-Smtp-Source: ABdhPJwB/h4hp6eiaPL7IIkokMwriJ92sqjBOp3ptJnp+mtkZNAr8JNrt86gahMU24MuTsB8HUnK9oEXGRKV+55XO5U= -X-Received: by 2002:a02:c916:: with SMTP id t22mr15608489jao.300.1643769379473; - Tue, 01 Feb 2022 18:36:19 -0800 (PST) -MIME-Version: 1.0 -From: Kamal Johnson -Date: Wed, 2 Feb 2022 08:06:06 +0530 -Message-ID: -Subject: Hfruuhhs -To: Kamal Johnson -Content-Type: multipart/alternative; boundary="000000000000f4cb7505d6ffe106" - ---000000000000f4cb7505d6ffe106 -Content-Type: text/plain; charset="UTF-8" - -Yysjtshsg - ---000000000000f4cb7505d6ffe106 -Content-Type: text/html; charset="UTF-8" - -
    Yysjtshsg
    - ---000000000000f4cb7505d6ffe106-- \ No newline at end of file From 8f8bafbdde33acbb0b71acb999ab683d40fdbe2b Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Fri, 4 Feb 2022 15:29:12 +0530 Subject: [PATCH 31/68] fix: sider styling issue --- frappe/email/doctype/email_account/email_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 6d84cbb270..2aa409628b 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -474,7 +474,7 @@ class EmailAccount(Document): if frappe.local.flags.in_test: if self.enable_incoming: for folder in self.imap_folder: - _messages = messages[folder.folder_name] if folder.folder_name in messages and ( messages[folder.folder_name] is not None) else {} + _messages = messages[folder.folder_name] if folder.folder_name in messages and (messages[folder.folder_name] is not None) else {} process_mail(_messages, folder.append_to) return mails else: From 1cc0f88d0b0194f73194cce7e99fddba017b7731 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Fri, 4 Feb 2022 15:47:27 +0530 Subject: [PATCH 32/68] fix: issue when test_mails are passed while testing --- frappe/email/doctype/email_account/email_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 2aa409628b..ddc3e349cd 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -472,7 +472,7 @@ class EmailAccount(Document): mails.append(InboundMail(message, self, uid, seen_status, append_to)) if frappe.local.flags.in_test: - if self.enable_incoming: + if self.enable_incoming and not test_mails: for folder in self.imap_folder: _messages = messages[folder.folder_name] if folder.folder_name in messages and (messages[folder.folder_name] is not None) else {} process_mail(_messages, folder.append_to) From c80c81ed329937f3295605b451f7d58a71b99cf6 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Fri, 4 Feb 2022 18:20:07 +0530 Subject: [PATCH 33/68] fix: messages and test_mail not passed issue while running test --- frappe/email/doctype/email_account/email_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index ddc3e349cd..cc7a1352cf 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -472,7 +472,7 @@ class EmailAccount(Document): mails.append(InboundMail(message, self, uid, seen_status, append_to)) if frappe.local.flags.in_test: - if self.enable_incoming and not test_mails: + if self.enable_incoming and not test_mails and messages: for folder in self.imap_folder: _messages = messages[folder.folder_name] if folder.folder_name in messages and (messages[folder.folder_name] is not None) else {} process_mail(_messages, folder.append_to) From 3f21b6707a1386c0366eb05efbf279cef2d43bbf Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Sun, 6 Feb 2022 08:40:23 +0530 Subject: [PATCH 34/68] fix: Use whoosh AsyncWriter to prevent write locks --- frappe/search/full_text_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py index 1d4f3fef32..79ccd3c6d5 100644 --- a/frappe/search/full_text_search.py +++ b/frappe/search/full_text_search.py @@ -66,7 +66,7 @@ class FullTextSearch: ix = self.get_index() with ix.searcher(): - writer = ix.writer() + writer = AsyncWriter(ix) writer.delete_by_term(self.id, doc_name) writer.commit(optimize=True) @@ -98,7 +98,7 @@ class FullTextSearch: def build_index(self): """Build index for all parsed documents""" ix = self.create_index() - writer = ix.writer() + writer = AsyncWriter(ix) for i, document in enumerate(self.documents): if document: From 1944a547f96dec83474390b5bce9b52371d989fd Mon Sep 17 00:00:00 2001 From: Wolfram Schmidt Date: Fri, 4 Feb 2022 17:41:08 +0100 Subject: [PATCH 35/68] added translation for Yes/No Since v13 the translation of the english No was "Kein". Which is not the right choice when selecting between yes or no. "Kein" is more something like none... --- frappe/translations/de.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index 9fad17becd..e3085d0b7d 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -151,6 +151,7 @@ My Account,Mein Konto, New Address,Neue Adresse, New Contact,Neuer Kontakt, Next,Weiter, +No,Nein, No Data,Keine Daten, No address added yet.,Noch keine Adresse hinzugefügt., No contacts added yet.,Noch keine Kontakte hinzugefügt., From ae2dd3f6b87d51e9446bd413667b0b82f83ee3da Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 7 Feb 2022 11:59:06 +0530 Subject: [PATCH 36/68] fix: Remove `in_test` flag inside `get_inbound_mails` - Use mocked data instead for testing --- .../doctype/email_account/email_account.py | 17 +++++----- .../email_account/test_email_account.py | 33 +++++++++++-------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index cc7a1352cf..424690e797 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -471,14 +471,14 @@ class EmailAccount(Document): # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' mails.append(InboundMail(message, self, uid, seen_status, append_to)) - if frappe.local.flags.in_test: - if self.enable_incoming and not test_mails and messages: - for folder in self.imap_folder: - _messages = messages[folder.folder_name] if folder.folder_name in messages and (messages[folder.folder_name] is not None) else {} - process_mail(_messages, folder.append_to) - return mails - else: - return [InboundMail(msg, self) for msg in test_mails or []] + # if frappe.local.flags.in_test: + # if self.enable_incoming and not test_mails and messages: + # for folder in self.imap_folder: + # _messages = messages[folder.folder_name] if folder.folder_name in messages and (messages[folder.folder_name] is not None) else {} + # process_mail(_messages, folder.append_to) + # return mails + # else: + # return [InboundMail(msg, self) for msg in test_mails or []] if not self.enable_incoming: return [] @@ -502,7 +502,6 @@ class EmailAccount(Document): except Exception: frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) return [] - return mails def handle_bad_emails(self, uid, raw, reason): diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 3555353259..4c1acf091b 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -14,11 +14,11 @@ from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments from frappe.email.doctype.email_account.email_account import notify_unreplied +from unittest.mock import patch + make_test_records("User") make_test_records("Email Account") - - class TestEmailAccount(unittest.TestCase): @classmethod def setUpClass(cls): @@ -257,15 +257,16 @@ class TestEmailAccount(unittest.TestCase): self.assertTrue(communication.reference_name) self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) - def test_append_to_with_imap_folders(self): - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) + @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) + # @patch("frappe.email.receive.EmailServer.get_messages", side_effect=get_mocked_messages) + def test_append_to_with_imap_folders(self, mocked_logout, mocked_select_imap_folder): mail_content_1 = self.get_test_mail(fname="incoming-1.raw") mail_content_2 = self.get_test_mail(fname="incoming-2.raw") mail_content_3 = self.get_test_mail(fname="incoming-3.raw") messages = { - 'INBOX': { # append_to = ToDo + '"INBOX"': { # append_to = ToDo 'latest_messages': [ mail_content_1, mail_content_2 @@ -276,7 +277,7 @@ class TestEmailAccount(unittest.TestCase): }, 'uid_list': [0,1] }, - 'Test Folder': { # append_to = Communication + '"Test Folder"': { # append_to = Communication 'latest_messages': [ mail_content_3 ], @@ -286,9 +287,15 @@ class TestEmailAccount(unittest.TestCase): 'uid_list': [2] } } - mails = email_account.get_inbound_mails(messages=messages) - self.assertEqual(len(mails), 3) - + from frappe.email.receive import EmailServer + def get_mocked_messages(**args): + return messages[args["folder"]] + + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + mails = email_account.get_inbound_mails(messages=messages) + self.assertEqual(len(mails), 3) + inbox_mails = 0 test_folder_mails = 0 @@ -373,11 +380,11 @@ class TestInboundMail(unittest.TestCase): email_account = frappe.get_doc("Email Account", "_Test Email Account 1") inbound_mail = InboundMail(mail_content, email_account, 12345, 1) - new_communiction = inbound_mail.process() + new_communication = inbound_mail.process() # Make sure that uid is changed to new uid - self.assertEqual(new_communiction.uid, 12345) - self.assertEqual(communication.name, new_communiction.name) + self.assertEqual(new_communication.uid, 12345) + self.assertEqual(communication.name, new_communication.name) def test_find_parent_email_queue(self): """If the mail is reply to the already sent mail, there will be a email queue record. From fb1ff60ab250729ac5ed5a553ed80d39ca05d70c Mon Sep 17 00:00:00 2001 From: Wolfram Schmidt Date: Mon, 7 Feb 2022 07:50:56 +0100 Subject: [PATCH 37/68] polished the "Tag" translation (#15889) corrected some translations and added missing ones --- frappe/translations/de.csv | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index e3085d0b7d..f682a51e17 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -350,7 +350,7 @@ Add a New Role,Neue Rolle hinzufügen, Add a column,Spalte einfügen, Add a comment,Einen Kommentar hinzufügen, Add a new section,Fügen Sie einen neuen Abschnitt hinzu, -Add a tag ...,Füge einen Tag hinzu ..., +Add a tag ...,Füge ein Schlagwort hinzu ..., Add all roles,Alle Rollen hinzufügen, Add custom forms.,Benutzerdefinierte Formulare hinzufügen, Add custom javascript to forms.,Benutzerdefiniertes Javascript zum Formular hinzufügen, @@ -947,6 +947,7 @@ Edit Auto Email Report Settings,Bearbeiten Sie die Einstellungen für automatisc Edit Custom HTML,Benutzerdefiniertes HTML bearbeiten, Edit DocType,DocType bearbeiten, Edit Filter,Filter bearbeiten, +Edit Filters,Filter bearbeiten, Edit Format,Format bearbeiten, Edit HTML,HTML bearbeiten, Edit Heading,Kopf bearbeiten, @@ -1231,6 +1232,7 @@ Hide Copy,Kopie ausblenden, Hide Footer Signup,Fußzeilen-Anmeldung ausblenden, Hide Sidebar and Menu,Seitenleiste und Menü ausblenden, Hide Standard Menu,Standardmenü ausblenden, +Hide Tags,Schlagworte ausblenden, Hide Weekends,Wochenenden ausblenden, Hide details,Details ausblenden, Hide footer in auto email reports,Fußzeile in automatischen E-Mail-Berichten ausblenden, @@ -1651,7 +1653,7 @@ No Preview,Keine Vorschau, No Preview Available,Keine Vorschau vorhanden, No Printer is Available.,Es ist kein Drucker verfügbar., No Results,Keine Ergebnisse, -No Tags,No Tags, +No Tags,Keine Schlagworte, No alerts for today,Keine Warnungen für heute, No comments yet,Noch keine Kommentare, No comments yet. Start a new discussion.,Noch keine Kommentare. Starten Sie eine neue Diskussion., @@ -2041,7 +2043,7 @@ Remove,Entfernen, Remove Field,Feld entfernen, Remove Filter,Filter entfernen, Remove Section,Abschnitt entfernen, -Remove Tag,Markierung entfernen, +Remove Tag,Schlagwort entfernen, Remove all customizations?,Alle Anpassungen entfernen?, Removed {0},{0} entfernt, Rename many items by uploading a .csv file.,Viele Elemente auf einmal umbenennen durch Hochladen einer .CSV-Datei, @@ -3251,7 +3253,7 @@ DocType Action,DocType-Aktion, DocType Event,DocType-Ereignis, DocType Link,DocType Link, Document Share,Dokumentenfreigabe, -Document Tag,Dokument-Tag, +Document Tag,Dokument-Schlagwort, Document Title,Dokumenttitel, Document Type Field Mapping,Dokumenttyp-Feldzuordnung, Document Type Mapping,Dokumenttypzuordnung, @@ -3797,7 +3799,7 @@ Start,Start, Start Time,Startzeit, Status,Status, Submitted,Gebucht, -Tag,Etikett, +Tag,Schlagwort, Template,Vorlage, Thursday,Donnerstag, Title,Bezeichnung, @@ -4029,7 +4031,7 @@ Please select target language for translation,Bitte wählen Sie die Zielsprache Select Language,Sprache auswählen, Confirm Translations,Übersetzungen bestätigen, Contributed Translations,Beigetragene Übersetzungen, -Show Tags,Tags anzeigen, +Show Tags,Schlagworte anzeigen, Do not have permission to access {0} bucket.,Sie haben keine Berechtigung zum Zugriff auf den Bucket {0}., Allow document creation via Email,Dokumenterstellung per E-Mail zulassen, Sender Field,Absenderfeld, From 1cd47222353accf9bcbf403182fb52fc84bdbf7e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 7 Feb 2022 14:00:26 +0530 Subject: [PATCH 38/68] fix: use `now` instead of `is_async` in tests (#15893) --- frappe/model/delete_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index afe01d9106..552a1e20d8 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -115,7 +115,7 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa # All the linked docs should be checked beforehand frappe.enqueue('frappe.model.delete_doc.delete_dynamic_links', doctype=doc.doctype, name=doc.name, - is_async=False if frappe.flags.in_test else True) + now=frappe.flags.in_test) # clear cache for Document doc.clear_cache() From a3043d8621e13478bb55bcc2699660ac17f22182 Mon Sep 17 00:00:00 2001 From: Rohan Bansal Date: Mon, 7 Feb 2022 14:49:16 +0530 Subject: [PATCH 39/68] fix: append chart options before render --- frappe/public/js/frappe/form/dashboard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index df4dbf09e7..6e3dd3eb0b 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -549,14 +549,14 @@ frappe.ui.form.Dashboard = class FormDashboard { render_graph(args) { this.chart_area.show(); this.chart_area.body.empty(); - $.extend({ + $.extend(args, { type: 'line', colors: ['green'], truncateLegends: 1, axisOptions: { shortenYAxisNumbers: 1 } - }, args); + }); this.show(); this.chart = new frappe.Chart('.form-graph', args); From b31f3c24f6af2aec39c0c176010ac1297201c82a Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 23 Dec 2021 12:40:21 +0530 Subject: [PATCH 40/68] refactor: remove parent, parenttype, parentfield, idx columns from non-child table doctypes * feat: add parent, parenttype, idx, parentfield columns to doctypes when transitioning from normal -> child table * fix: remove parent, parenttype, parentfield, idx from DocType DocType --- frappe/api.py | 3 +- frappe/client.py | 25 +++---- frappe/contacts/doctype/address/address.py | 2 +- frappe/contacts/doctype/contact/contact.py | 2 +- frappe/core/doctype/data_import/importer.py | 2 +- frappe/core/doctype/doctype/doctype.py | 24 ++++++- frappe/core/doctype/file/file.py | 5 +- frappe/core/doctype/report/report.py | 2 +- frappe/core/doctype/user_type/user_type.py | 2 +- frappe/database/database.py | 10 ++- frappe/database/mariadb/framework_mariadb.sql | 7 +- frappe/database/mariadb/schema.py | 16 +++-- .../database/postgres/framework_postgres.sql | 4 -- frappe/database/postgres/schema.py | 27 +++++--- frappe/database/schema.py | 47 +++++++++++--- frappe/desk/doctype/tag/tag.py | 2 - frappe/desk/form/linked_with.py | 48 ++++++++------ frappe/desk/reportview.py | 4 +- frappe/desk/search.py | 2 +- .../auto_email_report/auto_email_report.py | 2 +- .../document_type_mapping.py | 4 +- frappe/model/__init__.py | 7 +- frappe/model/base_document.py | 65 ++++++++++++------- frappe/model/delete_doc.py | 26 ++++---- frappe/model/document.py | 12 ++-- frappe/model/mapper.py | 5 +- frappe/model/meta.py | 28 +++++--- frappe/modules/export_file.py | 2 +- frappe/public/js/frappe/model/model.js | 4 +- frappe/tests/test_db_update.py | 11 ++-- frappe/utils/__init__.py | 3 - frappe/utils/data.py | 4 +- frappe/website/doctype/web_form/web_form.py | 2 +- frappe/website/website_generator.py | 1 - 34 files changed, 251 insertions(+), 159 deletions(-) diff --git a/frappe/api.py b/frappe/api.py index b061761d10..e7f7bf5a04 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -94,7 +94,8 @@ def handle(): "data": doc.save().as_dict() }) - if doc.parenttype and doc.parent: + # check for child table doctype + if doc.get("parenttype"): frappe.get_doc(doc.parenttype, doc.parent).save() frappe.db.commit() diff --git a/frappe/client.py b/frappe/client.py index 7280c29ba4..1898994afe 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -128,7 +128,7 @@ def set_value(doctype, name, fieldname, value=None): :param fieldname: fieldname string or JSON / dict with key value pair :param value: value if fieldname is JSON / dict''' - if fieldname!="idx" and fieldname in frappe.model.default_fields: + if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): frappe.throw(_("Cannot edit standard fields")) if not value: @@ -141,14 +141,15 @@ def set_value(doctype, name, fieldname, value=None): else: values = {fieldname: value} - doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) - if doc and doc.parent and doc.parenttype: + # check for child table doctype + if not frappe.get_meta(doctype).istable: + doc = frappe.get_doc(doctype, name) + doc.update(values) + else: + doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) doc = frappe.get_doc(doc.parenttype, doc.parent) child = doc.getone({"doctype": doctype, "name": name}) child.update(values) - else: - doc = frappe.get_doc(doctype, name) - doc.update(values) doc.save() @@ -162,10 +163,10 @@ def insert(doc=None): if isinstance(doc, str): doc = json.loads(doc) - if doc.get("parent") and doc.get("parenttype"): + if doc.get("parenttype"): # inserting a child record - parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent")) - parent.append(doc.get("parentfield"), doc) + parent = frappe.get_doc(doc.parenttype, doc.parent) + parent.append(doc.parentfield, doc) parent.save() return parent.as_dict() else: @@ -186,10 +187,10 @@ def insert_many(docs=None): frappe.throw(_('Only 200 inserts allowed in one request')) for doc in docs: - if doc.get("parent") and doc.get("parenttype"): + if doc.get("parenttype"): # inserting a child record - parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent")) - parent.append(doc.get("parentfield"), doc) + parent = frappe.get_doc(doc.parenttype, doc.parent) + parent.append(doc.parentfield, doc) parent.save() out.append(parent.name) else: diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 5d0ed18d5f..921990f2a3 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -241,7 +241,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): {mcond} {condition} order by if(locate(%(_txt)s, `tabAddress`.name), locate(%(_txt)s, `tabAddress`.name), 99999), - `tabAddress`.idx desc, `tabAddress`.name + `tabAddress`.name limit %(start)s, %(page_len)s """.format( mcond=get_match_cond(doctype), key=searchfield, diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index 9152655b85..d3d0fba62c 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -215,7 +215,7 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters): {mcond} order by if(locate(%(_txt)s, `tabContact`.name), locate(%(_txt)s, `tabContact`.name), 99999), - `tabContact`.idx desc, `tabContact`.name + `tabContact`.name limit %(start)s, %(page_len)s """.format(mcond=get_match_cond(doctype), key=searchfield), { 'txt': '%' + txt + '%', '_txt': txt.replace("%", ""), diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 107c05a66a..f085709945 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -618,7 +618,7 @@ class Row: ) # remove standard fields and __islocal - for key in frappe.model.default_fields + ("__islocal",): + for key in frappe.model.default_fields + frappe.model.child_table_fields + ("__islocal",): doc.pop(key, None) for col, value in zip(columns, values): diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 67c31b704d..2df7516d93 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -10,7 +10,9 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache import frappe from frappe import _ from frappe.utils import now, cint -from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options +from frappe.model import ( + no_value_fields, default_fields, table_fields, data_field_options, child_table_fields +) from frappe.model.document import Document from frappe.model.base_document import get_controller from frappe.custom.doctype.property_setter.property_setter import make_property_setter @@ -74,6 +76,7 @@ class DocType(Document): self.make_amendable() self.make_repeatable() self.validate_nestedset() + self.validate_child_table() self.validate_website() self.ensure_minimum_max_attachment_limit() validate_links_table_fieldnames(self) @@ -689,6 +692,23 @@ class DocType(Document): }) self.nsm_parent_field = parent_field_name + def validate_child_table(self): + if not self.get("istable") or self.is_new(): + # if the doctype is not a child table then return + # if the doctype is a new doctype and also a child table then + # don't move forward as it will be handled via schema + return + + self.add_child_table_fields() + + def add_child_table_fields(self): + from frappe.database.schema import add_column + + add_column(self.name, "parent", "Data") + add_column(self.name, "parenttype", "Data") + add_column(self.name, "parentfield", "Data") + add_column(self.name, "idx", "Int", length=8, not_null=True, default="0") + def get_max_idx(self): """Returns the highest `idx`""" max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", @@ -1016,7 +1036,7 @@ def validate_fields(meta): sort_fields = [d.split()[0] for d in meta.sort_field.split(',')] for fieldname in sort_fields: - if not fieldname in fieldname_list + list(default_fields): + if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)): frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname), InvalidFieldNameError) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index ee2c9987b6..2808a2710b 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -878,8 +878,9 @@ def extract_images_from_html(doc, content, is_private=False): else: filename = get_random_filename(content_type=mtype) - doctype = doc.parenttype if doc.parent else doc.doctype - name = doc.parent or doc.name + # attaching a file to a child table doc, attaches it to the parent doc + doctype = doc.parenttype if doc.get("parent") else doc.doctype + name = doc.get("parent") or doc.name _file = frappe.get_doc({ "doctype": "File", diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 266017dd71..9cb40dffd4 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -61,7 +61,7 @@ class Report(Document): delete_permanently=True) def get_columns(self): - return [d.as_dict(no_default_fields = True) for d in self.columns] + return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns] @frappe.whitelist() def set_doctype_roles(self): diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 626ab772b8..a64ad1a130 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters ['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]] doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters, - order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1) + limit_start=start, limit_page_length=page_len, as_list=1) custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)], ['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']] diff --git a/frappe/database/database.py b/frappe/database/database.py index 8a6b83c5d9..3cebe4510d 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -37,9 +37,9 @@ class Database(object): OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"] DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"] - STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype') - DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent', - 'parentfield', 'parenttype', 'idx'] + STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by') + DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus'] + CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield', 'idx') MAX_WRITES_PER_TRANSACTION = 200_000 class InvalidColumnName(frappe.ValidationError): pass @@ -435,11 +435,9 @@ class Database(object): else: fields = fieldname - if fieldname!="*": + if fieldname != "*": if isinstance(fieldname, str): fields = [fieldname] - else: - fields = fieldname if (filters is not None) and (filters!=doctype or doctype=="DocType"): try: diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index cfb4e243a2..b59beefa0b 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -171,10 +171,6 @@ CREATE TABLE `tabDocType` ( `modified_by` varchar(255) DEFAULT NULL, `owner` varchar(255) DEFAULT NULL, `docstatus` int(1) NOT NULL DEFAULT 0, - `parent` varchar(255) DEFAULT NULL, - `parentfield` varchar(255) DEFAULT NULL, - `parenttype` varchar(255) DEFAULT NULL, - `idx` int(8) NOT NULL DEFAULT 0, `search_fields` varchar(255) DEFAULT NULL, `issingle` int(1) NOT NULL DEFAULT 0, `is_tree` int(1) NOT NULL DEFAULT 0, @@ -228,8 +224,7 @@ CREATE TABLE `tabDocType` ( `subject_field` varchar(255) DEFAULT NULL, `sender_field` varchar(255) DEFAULT NULL, `migration_hash` varchar(255) DEFAULT NULL, - PRIMARY KEY (`name`), - KEY `parent` (`parent`) + PRIMARY KEY (`name`) ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 07bb4d5d7c..2f375bc751 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -18,6 +18,17 @@ class MariaDBTable(DBTable): if index_defs: additional_definitions += ',\n'.join(index_defs) + ',\n' + # child table columns + if self.meta.get("istable") or 0: + additional_definitions += ',\n'.join( + ( + f"parent varchar({varchar_len})", + f"parentfield varchar({varchar_len})", + f"parenttype varchar({varchar_len})", + "idx int(8) not null default '0'" + ) + ) + ',\n' + # create table query = f"""create table `{self.table_name}` ( name varchar({varchar_len}) not null primary key, @@ -26,12 +37,7 @@ class MariaDBTable(DBTable): modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus int(1) not null default '0', - parent varchar({varchar_len}), - parentfield varchar({varchar_len}), - parenttype varchar({varchar_len}), - idx int(8) not null default '0', {additional_definitions} - index parent(parent), index modified(modified)) ENGINE={engine} ROW_FORMAT=DYNAMIC diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index f911e34650..19381607d4 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -176,10 +176,6 @@ CREATE TABLE "tabDocType" ( "modified_by" varchar(255) DEFAULT NULL, "owner" varchar(255) DEFAULT NULL, "docstatus" smallint NOT NULL DEFAULT 0, - "parent" varchar(255) DEFAULT NULL, - "parentfield" varchar(255) DEFAULT NULL, - "parenttype" varchar(255) DEFAULT NULL, - "idx" bigint NOT NULL DEFAULT 0, "search_fields" varchar(255) DEFAULT NULL, "issingle" smallint NOT NULL DEFAULT 0, "is_tree" smallint NOT NULL DEFAULT 0, diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index a2d5be0b70..9dbb597adf 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -5,26 +5,37 @@ from frappe.database.schema import DBTable, get_definition class PostgresTable(DBTable): def create(self): - add_text = '' + add_text = "" # columns column_defs = self.get_column_definitions() - if column_defs: add_text += ',\n'.join(column_defs) + if column_defs: + add_text += ",\n".join(column_defs) + + # child table columns + if self.meta.get("istable") or 0: + if column_defs: + add_text += ",\n" + + add_text += ",\n".join( + ( + "parent varchar({varchar_len})", + "parentfield varchar({varchar_len})", + "parenttype varchar({varchar_len})", + "idx bigint not null default '0'" + ) + ) # TODO: set docstatus length # create table - frappe.db.sql("""create table `%s` ( + frappe.db.sql(("""create table `%s` ( name varchar({varchar_len}) not null primary key, creation timestamp(6), modified timestamp(6), modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus smallint not null default '0', - parent varchar({varchar_len}), - parentfield varchar({varchar_len}), - parenttype varchar({varchar_len}), - idx bigint not null default '0', - %s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text)) + %s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN)) self.create_indexes() frappe.db.commit() diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 9a6dd502dc..7315d3630a 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -106,6 +106,9 @@ class DBTable: columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in frappe.db.STANDARD_VARCHAR_COLUMNS] + if self.meta.get("istable"): + columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in + frappe.db.CHILD_TABLE_COLUMNS if f != "idx"] columns += self.columns.values() for col in columns: @@ -300,11 +303,12 @@ def validate_column_length(fieldname): def get_definition(fieldtype, precision=None, length=None): d = frappe.db.type_map.get(fieldtype) - # convert int to long int if the length of the int is greater than 11 - if fieldtype == "Int" and length and length > 11: - d = frappe.db.type_map.get("Long Int") + if not d: + return - if not d: return + if fieldtype == "Int" and length and length > 11: + # convert int to long int if the length of the int is greater than 11 + d = frappe.db.type_map.get("Long Int") coltype = d[0] size = d[1] if d[1] else None @@ -315,19 +319,44 @@ def get_definition(fieldtype, precision=None, length=None): if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6: size = '21,9' - if coltype == "varchar" and length: - size = length + if length: + if coltype == "varchar": + size = length + elif coltype == "int" and length < 11: + # allow setting custom length for int if length provided is less than 11 + # NOTE: this will only be applicable for mariadb as frappe implements int + # in postgres as bigint (as seen in type_map) + size = length if size is not None: coltype = "{coltype}({size})".format(coltype=coltype, size=size) return coltype -def add_column(doctype, column_name, fieldtype, precision=None): +def add_column( + doctype, + column_name, + fieldtype, + precision=None, + length=None, + default=None, + not_null=False +): if column_name in frappe.db.get_table_columns(doctype): # already exists return frappe.db.commit() - frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype, - column_name, get_definition(fieldtype, precision))) + + query = "alter table `tab%s` add column %s %s" % ( + doctype, + column_name, + get_definition(fieldtype, precision, length) + ) + + if not_null: + query += " not null" + if default: + query += f" default '{default}'" + + frappe.db.sql(query) diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 381c24a765..d44c481210 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -148,8 +148,6 @@ def update_tags(doc, tags): "doctype": "Tag Link", "document_type": doc.doctype, "document_name": doc.name, - "parenttype": doc.doctype, - "parent": doc.name, "title": doc.get_title() or '', "tag": tag }).insert(ignore_permissions=True) diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index cd87c898d8..572d3f2a94 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -389,8 +389,6 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): else: return results - me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) - for dt, link in linkinfo.items(): filters = [] link["doctype"] = dt @@ -413,11 +411,16 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters")) elif link.get("get_parent"): - if me and me.parent and me.parenttype == dt: + ret = None + + # check for child table + if not frappe.get_meta(doctype).istable: + continue + + me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) + if me and me.parenttype == dt: ret = frappe.get_all(doctype=dt, fields=fields, filters=[[dt, "name", '=', me.parent]]) - else: - ret = None elif link.get("child_doctype"): or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")] @@ -473,7 +476,7 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled)) ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled)) - filters=[['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]] + filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters) @@ -498,12 +501,12 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): - filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]] + filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1) - links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1) + links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1) ret = {} @@ -529,34 +532,37 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False): ret = {} - filters=[['fieldtype','=', 'Dynamic Link']] + filters = [['fieldtype','=', 'Dynamic Link']] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find dynamic links of parents links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) - links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) + links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) for df in links: if is_single(df.doctype): continue - # optimized to get both link exists and parenttype - possible_link = frappe.get_all(df.doctype, filters={df.doctype_fieldname: doctype}, - fields=['parenttype'], distinct=True) + is_child = frappe.get_meta(df.doctype).istable + possible_link = frappe.get_all( + df.doctype, + filters={df.doctype_fieldname: doctype}, + fields=["parenttype"] if is_child else None, + distinct=True + ) if not possible_link: continue - for d in possible_link: - # is child - if d.parenttype: + if is_child: + for d in possible_link: ret[d.parenttype] = { "child_doctype": df.doctype, "fieldname": [df.fieldname], "doctype_fieldname": df.doctype_fieldname } - else: - ret[df.doctype] = { - "fieldname": [df.fieldname], - "doctype_fieldname": df.doctype_fieldname - } + else: + ret[df.doctype] = { + "fieldname": [df.fieldname], + "doctype_fieldname": df.doctype_fieldname + } return ret diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 4001d0b9cf..c45fc9bfdd 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -6,7 +6,7 @@ import frappe, json import frappe.permissions from frappe.model.db_query import DatabaseQuery -from frappe.model import default_fields, optional_fields +from frappe.model import default_fields, optional_fields, child_table_fields from frappe import _ from io import StringIO from frappe.core.doctype.access_log.access_log import make_access_log @@ -156,7 +156,7 @@ def raise_invalid_field(fieldname): def is_standard(fieldname): if '.' in fieldname: parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None) - return fieldname in default_fields or fieldname in optional_fields + return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields def extract_fieldname(field): for text in (',', '/*', '#'): diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 95397070ae..e89ef2be24 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -147,7 +147,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, from frappe.model.db_query import get_order_by order_by_based_on_meta = get_order_by(doctype, meta) # 2 is the index of _relevance column - order_by = "_relevance, {0}, `tab{1}`.idx desc".format(order_by_based_on_meta, doctype) + order_by = "_relevance, {0}".format(order_by_based_on_meta) ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read' ignore_permissions = True if doctype == "DocType" else (cint(ignore_user_permissions) and has_permission(doctype, ptype=ptype)) diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index 34728375cd..682f0df7cf 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -252,7 +252,7 @@ def make_links(columns, data): if col.options and row.get(col.options): row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname]) elif col.fieldtype == "Currency": - doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.parent else None + doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.get("parent") else None # Pass the Document to get the currency based on docfield option row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc) return columns, data diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py index 8f1e5504da..0565b3219d 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py @@ -5,7 +5,7 @@ import frappe import json from frappe import _ from frappe.model.document import Document -from frappe.model import default_fields +from frappe.model import default_fields, child_table_fields class DocumentTypeMapping(Document): def validate(self): @@ -14,7 +14,7 @@ class DocumentTypeMapping(Document): def validate_inner_mapping(self): meta = frappe.get_meta(self.local_doctype) for field_map in self.field_mapping: - if field_map.local_fieldname not in default_fields: + if field_map.local_fieldname not in (default_fields + child_table_fields): field = meta.get_field(field_map.local_fieldname) if not field: frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx)) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index b50a0304a5..d635b31900 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -90,11 +90,14 @@ default_fields = ( 'creation', 'modified', 'modified_by', + 'docstatus' +) + +child_table_fields = ( 'parent', 'parentfield', 'parenttype', - 'idx', - 'docstatus' + 'idx' ) optional_fields = ( diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 94f2c5ea18..bd3875cbea 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -4,7 +4,7 @@ import frappe import datetime from frappe import _ -from frappe.model import default_fields, table_fields +from frappe.model import default_fields, table_fields, child_table_fields from frappe.model.naming import set_new_name from frappe.model.utils.link_count import notify_link_count from frappe.modules import load_doctype_module @@ -104,6 +104,10 @@ class BaseDocument(object): "balance": 42000 }) """ + + # QUESTION: why do we need the 1st for loop? + # we're essentially setting the values in d, in the 2nd for loop (?) + # first set default field values of base document for key in default_fields: if key in d: @@ -208,7 +212,10 @@ class BaseDocument(object): raise ValueError def remove(self, doc): - self.get(doc.parentfield).remove(doc) + # Usage: from the parent doc, pass the child table doc + # to remove that child doc from the child table, thus removing it from the parent doc + if doc.get("parentfield"): + self.get(doc.parentfield).remove(doc) def _init_child(self, value, key): if not self.doctype: @@ -285,16 +292,16 @@ class BaseDocument(object): if key not in self.__dict__: self.__dict__[key] = None - if self.__dict__[key] is None: - if key == "docstatus": - self.docstatus = DocStatus.draft() - elif key == "idx": - self.__dict__[key] = 0 + if key == "docstatus" and self.__dict__[key] is None: + self.__dict__[key] = DocStatus.draft() for key in self.get_valid_columns(): if key not in self.__dict__: self.__dict__[key] = None + if key == "idx" and self.__dict__[key] is None: + self.__dict__[key] = 0 + def get_valid_columns(self): if self.doctype not in frappe.local.valid_columns: if self.doctype in DOCTYPES_FOR_DOCTYPE: @@ -318,12 +325,19 @@ class BaseDocument(object): def docstatus(self, value): self.__dict__["docstatus"] = DocStatus(cint(value)) - def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False): + def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False): doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str) doc["doctype"] = self.doctype for df in self.meta.get_table_fields(): children = self.get(df.fieldname) or [] - doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children] + doc[df.fieldname] = [ + d.as_dict( + convert_dates_to_str=convert_dates_to_str, + no_nulls=no_nulls, + no_default_fields=no_default_fields, + no_child_table_fields=no_child_table_fields + ) for d in children + ] if no_nulls: for k in list(doc): @@ -335,6 +349,11 @@ class BaseDocument(object): if k in default_fields: del doc[k] + if no_child_table_fields: + for k in list(doc): + if k in child_table_fields: + del doc[k] + for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"): if self.get(key): doc[key] = self.get(key) @@ -514,12 +533,12 @@ class BaseDocument(object): if df.fieldtype in table_fields: return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label)) - elif self.parentfield: + # check if parentfield exists (only applicable for child table doctype) + elif self.get("parentfield"): return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)), _("Row"), self.idx, _("Value missing for"), _(df.label)) - else: - return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label)) + return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label)) missing = [] @@ -538,10 +557,11 @@ class BaseDocument(object): def get_invalid_links(self, is_submittable=False): """Returns list of invalid links and also updates fetch values if not set""" def get_msg(df, docname): - if self.parentfield: + # check if parentfield exists (only applicable for child table doctype) + if self.get("parentfield"): return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname) - else: - return "{}: {}".format(_(df.label), docname) + + return "{}: {}".format(_(df.label), docname) invalid_links = [] cancelled_links = [] @@ -615,11 +635,8 @@ class BaseDocument(object): fetch_from_fieldname = df.fetch_from.split('.')[-1] value = values[fetch_from_fieldname] if df.fieldtype in ['Small Text', 'Text', 'Data']: - if fetch_from_fieldname in default_fields: - from frappe.model.meta import get_default_df - fetch_from_df = get_default_df(fetch_from_fieldname) - else: - fetch_from_df = frappe.get_meta(doctype).get_field(fetch_from_fieldname) + from frappe.model.meta import get_default_df + fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname) if not fetch_from_df: frappe.throw( @@ -754,9 +771,9 @@ class BaseDocument(object): def throw_length_exceeded_error(self, df, max_length, value): - if self.parentfield and self.idx: + # check if parentfield exists (only applicable for child table doctype) + if self.get("parentfield"): reference = _("{0}, Row {1}").format(_(self.doctype), self.idx) - else: reference = "{0} {1}".format(_(self.doctype), self.name) @@ -867,7 +884,7 @@ class BaseDocument(object): :param parentfield: If fieldname is in child table.""" from frappe.model.meta import get_field_precision - if parentfield and not isinstance(parentfield, str): + if parentfield and not isinstance(parentfield, str) and parentfield.get("parentfield"): parentfield = parentfield.parentfield cache_key = parentfield or "main" @@ -894,7 +911,7 @@ class BaseDocument(object): from frappe.utils.formatters import format_value df = self.meta.get_field(fieldname) - if not df and fieldname in default_fields: + if not df: from frappe.model.meta import get_default_df df = get_default_df(fieldname) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 552a1e20d8..a888e3f7f7 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -222,32 +222,36 @@ def check_if_doc_is_linked(doc, method="Delete"): """ from frappe.model.rename_doc import get_link_fields link_fields = get_link_fields(doc.doctype) - link_fields = [[lf['parent'], lf['fieldname'], lf['issingle']] for lf in link_fields] + ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + + for lf in link_fields: + link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle'] - for link_dt, link_field, issingle in link_fields: if not issingle: - for item in frappe.db.get_values(link_dt, {link_field:doc.name}, - ["name", "parent", "parenttype", "docstatus"], as_dict=True): - linked_doctype = item.parenttype if item.parent else link_dt + fields = ["name", "docstatus"] + if frappe.get_meta(link_dt).istable: + fields.extend(["parent", "parenttype"]) - ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + # NOTE: scenario: parent doc <-(linked to) child table doc <-(linked to) doc + for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True): + # available only in child table cases + item_parent = getattr(item, "parent", None) + linked_doctype = item.parenttype if item_parent else link_dt if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'): # don't check for communication and todo! continue - if not item: - continue - elif method != "Delete" and (method != "Cancel" or item.docstatus != 1): + if method != "Delete" and (method != "Cancel" or item.docstatus != 1): # don't raise exception if not # linked to a non-cancelled doc when deleting or to a submitted doc when cancelling continue - elif link_dt == doc.doctype and (item.parent or item.name) == doc.name: + elif link_dt == doc.doctype and (item_parent or item.name) == doc.name: # don't raise exception if not # linked to same item or doc having same name as the item continue else: - reference_docname = item.parent or item.name + reference_docname = item_parent or item.name raise_link_exists_exception(doc, linked_doctype, reference_docname) else: diff --git a/frappe/model/document.py b/frappe/model/document.py index 7b6b212ebc..f7ba9250fa 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -527,7 +527,7 @@ class Document(BaseDocument): def _validate_non_negative(self): def get_msg(df): - if self.parentfield: + if self.get("parentfield"): return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)), _("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label))) else: @@ -1202,7 +1202,7 @@ class Document(BaseDocument): if not frappe.compare(val1, condition, val2): label = doc.meta.get_label(fieldname) condition_str = error_condition_map.get(condition, condition) - if doc.parentfield: + if doc.get("parentfield"): msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2) else: msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2) @@ -1226,7 +1226,7 @@ class Document(BaseDocument): doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]})) for fieldname in fieldnames: - doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield))) + doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield")))) def get_url(self): """Returns Desk URL for this document.""" @@ -1379,9 +1379,11 @@ class Document(BaseDocument): doctype = self.__class__.__name__ docstatus = f" docstatus={self.docstatus}" if self.docstatus else "" - parent = f" parent={self.parent}" if self.parent else "" + repr_str = f"<{doctype}: {name}{docstatus}" - return f"<{doctype}: {name}{docstatus}{parent}>" + if not hasattr(self, "parent"): + return repr_str + ">" + return f"{repr_str} parent={self.parent}>" def __str__(self): name = self.name or "unsaved" diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index bde4fb6d73..1ebff84a08 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -4,7 +4,7 @@ import json import frappe from frappe import _ -from frappe.model import default_fields, table_fields +from frappe.model import default_fields, table_fields, child_table_fields from frappe.utils import cstr @@ -149,6 +149,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent): no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] + [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] + list(default_fields) + + list(child_table_fields) + list(table_map.get("field_no_map", []))) for df in target_doc.meta.get("fields"): @@ -183,7 +184,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent): target_doc.set(fmap[1], val) # map idx - if source_doc.idx: + if source_doc.get("idx"): target_doc.idx = source_doc.idx # add fetch diff --git a/frappe/model/meta.py b/frappe/model/meta.py index a483f3f2d6..ef8eca3629 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -18,7 +18,7 @@ from datetime import datetime import click import frappe, json, os from frappe.utils import cstr, cint, cast -from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields +from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields, child_table_fields from frappe.model.document import Document from frappe.model.base_document import BaseDocument from frappe.modules import load_doctype_module @@ -191,6 +191,8 @@ class Meta(Document): else: self._valid_columns = self.default_fields + \ [df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes] + if self.istable: + self._valid_columns += list(child_table_fields) return self._valid_columns @@ -520,7 +522,7 @@ class Meta(Document): '''add `links` child table in standard link dashboard format''' dashboard_links = [] - if hasattr(self, 'links') and self.links: + if getattr(self, 'links', None): dashboard_links.extend(self.links) if not data.transactions: @@ -625,9 +627,9 @@ def get_field_currency(df, doc=None): frappe.local.field_currency = frappe._dict() if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or - (doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))): + (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))): - ref_docname = doc.parent or doc.name + ref_docname = doc.get("parent") or doc.name if ":" in cstr(df.get("options")): split_opts = df.get("options").split(":") @@ -635,7 +637,7 @@ def get_field_currency(df, doc=None): currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2]) else: currency = doc.get(df.get("options")) - if doc.parent: + if doc.get("parenttype"): if currency: ref_docname = doc.name else: @@ -648,7 +650,7 @@ def get_field_currency(df, doc=None): .setdefault(df.fieldname, currency) return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \ - (doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname)) + (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname)) def get_field_precision(df, doc=None, currency=None): """get precision based on DocField options and fieldvalue in doc""" @@ -669,19 +671,25 @@ def get_field_precision(df, doc=None, currency=None): def get_default_df(fieldname): - if fieldname in default_fields: + if fieldname in (default_fields + child_table_fields): if fieldname in ("creation", "modified"): return frappe._dict( fieldname = fieldname, fieldtype = "Datetime" ) - else: + elif fieldname == "idx": return frappe._dict( fieldname = fieldname, - fieldtype = "Data" + fieldtype = "Int" ) + return frappe._dict( + fieldname = fieldname, + fieldtype = "Data" + ) + + def trim_tables(doctype=None, dry_run=False, quiet=False): """ Removes database fields that don't exist in the doctype (json or custom field). This may be needed @@ -713,7 +721,7 @@ def trim_tables(doctype=None, dry_run=False, quiet=False): def trim_table(doctype, dry_run=True): frappe.cache().hdel('table_columns', f"tab{doctype}") - ignore_fields = default_fields + optional_fields + ignore_fields = default_fields + optional_fields + child_table_fields columns = frappe.db.get_table_columns(doctype) fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value() is_internal = lambda f: f not in ignore_fields and not f.startswith("_") diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py index ab6ffd4985..45e008fa04 100644 --- a/frappe/modules/export_file.py +++ b/frappe/modules/export_file.py @@ -47,7 +47,7 @@ def strip_default_fields(doc, doc_export): for df in doc.meta.get_table_fields(): for d in doc_export.get(df.fieldname): - for fieldname in frappe.model.default_fields: + for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): if fieldname in d: del d[fieldname] diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index 041905408a..ef161fca3d 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -10,8 +10,7 @@ $.extend(frappe.model, { layout_fields: ['Section Break', 'Column Break', 'Tab Break', 'Fold'], std_fields_list: ['name', 'owner', 'creation', 'modified', 'modified_by', - '_user_tags', '_comments', '_assign', '_liked_by', 'docstatus', - 'parent', 'parenttype', 'parentfield', 'idx'], + '_user_tags', '_comments', '_assign', '_liked_by', 'docstatus'], core_doctypes_list: ['DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role', 'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form', @@ -20,7 +19,6 @@ $.extend(frappe.model, { std_fields: [ {fieldname:'name', fieldtype:'Link', label:__('ID')}, {fieldname:'owner', fieldtype:'Link', label:__('Created By'), options: 'User'}, - {fieldname:'idx', fieldtype:'Int', label:__('Index')}, {fieldname:'creation', fieldtype:'Date', label:__('Created On')}, {fieldname:'modified', fieldtype:'Date', label:__('Last Updated On')}, {fieldname:'modified_by', fieldtype:'Data', label:__('Last Updated By')}, diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index d2c54ef18c..4c0cc4b2bc 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -103,13 +103,9 @@ def get_other_fields_meta(meta): default_fields_map = { 'name': ('Data', 0), 'owner': ('Data', 0), - 'parent': ('Data', 0), - 'parentfield': ('Data', 0), 'modified_by': ('Data', 0), - 'parenttype': ('Data', 0), 'creation': ('Datetime', 0), 'modified': ('Datetime', 0), - 'idx': ('Int', 8), 'docstatus': ('Check', 0) } @@ -117,8 +113,13 @@ def get_other_fields_meta(meta): if meta.track_seen: optional_fields.append('_seen') + child_table_fields_map = {} + if meta.istable: + child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS if field != 'idx'}) + child_table_fields_map.update({'idx': ('Int', 8)}) + optional_fields_map = {field: ('Text', 0) for field in optional_fields} - fields = dict(default_fields_map, **optional_fields_map) + fields = dict(default_fields_map, **optional_fields_map, **child_table_fields_map) field_map = [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()] return field_map diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 6b93a81b6e..141adb9ea6 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -24,9 +24,6 @@ import frappe from frappe.utils.data import * from frappe.utils.html_utils import sanitize_html -default_fields = ['doctype', 'name', 'owner', 'creation', 'modified', 'modified_by', - 'parent', 'parentfield', 'parenttype', 'idx', 'docstatus'] - def get_fullname(user=None): """get the full name (first name + last name) of the user from User""" diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 34ddc23155..50c71bdc2e 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1362,7 +1362,7 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) - "fieldtype": } """ - from frappe.model import default_fields, optional_fields + from frappe.model import default_fields, optional_fields, child_table_fields if isinstance(f, dict): key, value = next(iter(f.items())) @@ -1400,7 +1400,7 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) - frappe.throw(frappe._("Operator must be one of {0}").format(", ".join(valid_operators))) - if f.doctype and (f.fieldname not in default_fields + optional_fields): + if f.doctype and (f.fieldname not in default_fields + optional_fields + child_table_fields): # verify fieldname belongs to the doctype meta = frappe.get_meta(f.doctype) if not meta.has_field(f.fieldname): diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 2e4d7a247b..8727443136 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -77,7 +77,7 @@ class WebForm(WebsiteGenerator): for prop in docfield_properties: if df.fieldtype==meta_df.fieldtype and prop not in ("idx", - "reqd", "default", "description", "default", "options", + "reqd", "default", "description", "options", "hidden", "read_only", "label"): df.set(prop, meta_df.get(prop)) diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py index e66496aa89..67b8e77c4f 100644 --- a/frappe/website/website_generator.py +++ b/frappe/website/website_generator.py @@ -118,7 +118,6 @@ class WebsiteGenerator(Document): "doc": self, "page_or_generator": "Generator", "ref_doctype":self.doctype, - "idx": self.idx, "docname": self.name, "controller": get_module_name(self.doctype, self.meta.module), }) From dfb1b8b406e8a8de3392ad6b5fc483bac7d7c88d Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 31 Jan 2022 15:15:16 +0530 Subject: [PATCH 41/68] chore(notifications.py): remove unused function get_unread_emails --- frappe/core/notifications.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index be3e723af6..414563e4c5 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -54,28 +54,3 @@ def get_unseen_likes(): & (comment_doctype.seen == 0) ) ) - - -def get_unread_emails(): - "returns count of unread emails for a user" - - communication_doctype = DocType("Communication") - user_doctype = DocType("User") - distinct_email_accounts = ( - frappe.qb.from_(user_doctype) - .select(user_doctype.email_account) - .where(user_doctype.parent == frappe.session.user) - .distinct() - ) - - return frappe.db.count(communication_doctype, - filters=( - (communication_doctype.communication_type == "Communication") - & (communication_doctype.communication_medium == "Email") - & (communication_doctype.sent_or_received == "Received") - & (communication_doctype.email_status.notin(["spam", "Trash"])) - & (communication_doctype.email_account.isin(distinct_email_accounts)) - & (communication_doctype.modified >= Now() - Interval(years=1)) - & (communication_doctype.seen == 0) - ) - ) From 9e42d1b769d33eb8da4df344aaca47b8aef5f731 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 31 Jan 2022 18:09:28 +0530 Subject: [PATCH 42/68] chore: remove patch --- frappe/patches.txt | 1 - ...ove_parent_and_parenttype_from_print_formats.py | 14 -------------- 2 files changed, 15 deletions(-) delete mode 100644 frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py diff --git a/frappe/patches.txt b/frappe/patches.txt index 7c2c6d5dc5..db9610a767 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -119,7 +119,6 @@ execute:frappe.delete_doc_if_exists('DocType', 'GSuite Settings') execute:frappe.delete_doc_if_exists('DocType', 'GSuite Templates') execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Account') execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings') -frappe.patches.v12_0.remove_parent_and_parenttype_from_print_formats frappe.patches.v12_0.remove_example_email_thread_notify execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders() frappe.patches.v12_0.set_correct_url_in_files diff --git a/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py b/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py deleted file mode 100644 index 1a3c56da59..0000000000 --- a/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py +++ /dev/null @@ -1,14 +0,0 @@ -import frappe - -def execute(): - frappe.db.sql(""" - UPDATE - `tabPrint Format` - SET - `tabPrint Format`.`parent`='', - `tabPrint Format`.`parenttype`='', - `tabPrint Format`.parentfield='' - WHERE - `tabPrint Format`.parent != '' - OR `tabPrint Format`.parenttype != '' - """) \ No newline at end of file From f64b0eee4601afcfdd2d061e30fd2f1018d90b9d Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 1 Feb 2022 13:41:19 +0530 Subject: [PATCH 43/68] test: child table creation and transitioning --- frappe/tests/test_child_table.py | 66 ++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 frappe/tests/test_child_table.py diff --git a/frappe/tests/test_child_table.py b/frappe/tests/test_child_table.py new file mode 100644 index 0000000000..8cdfd08599 --- /dev/null +++ b/frappe/tests/test_child_table.py @@ -0,0 +1,66 @@ +import frappe +from frappe.model import child_table_fields + +import unittest +from typing import Callable + + +class TestChildTable(unittest.TestCase): + def tearDown(self) -> None: + try: + frappe.delete_doc("DocType", self.doctype_name, force=1) + except Exception: + pass + + def test_child_table_doctype_creation_and_transitioning(self) -> None: + ''' + This method tests the creation of child table doctype + as well as it's transitioning from child table to normal and normal to child table doctype + ''' + + self.doctype_name = "Test Newy Child Table" + + try: + doc = frappe.get_doc({ + "doctype": "DocType", + "name": self.doctype_name, + "istable": 1, + "custom": 1, + "module": "Integrations", + "fields": [{ + "label": "Some Field", + "fieldname": "some_fieldname", + "fieldtype": "Data", + "reqd": 1 + }] + }).insert(ignore_permissions=True) + except Exception: + self.fail("Not able to create Child Table Doctype") + + + for column in child_table_fields: + self.assertTrue(frappe.db.has_column(self.doctype_name, column)) + + # check transitioning from child table to normal doctype + doc.istable = 0 + try: + doc.save(ignore_permissions=True) + except Exception: + self.fail("Not able to transition from Child Table Doctype to Normal Doctype") + + self.check_valid_columns(self.assertFalse) + + # check transitioning from normal to child table doctype + doc.istable = 1 + try: + doc.save(ignore_permissions=True) + except Exception: + self.fail("Not able to transition from Normal Doctype to Child Table Doctype") + + self.check_valid_columns(self.assertTrue) + + + def check_valid_columns(self, assertion_method: Callable) -> None: + valid_columns = frappe.get_meta(self.doctype_name).get_valid_columns() + for column in child_table_fields: + assertion_method(column in valid_columns) From aeadcb132251293dca38da79faa01f0803bbe438 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 2 Feb 2022 21:52:07 +0530 Subject: [PATCH 44/68] chore(notifications.py): remove unused get_unseen_likes --- frappe/core/notifications.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index 414563e4c5..5f41f217f0 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -39,18 +39,3 @@ def get_todays_events(as_list=False): today = nowdate() events = get_events(today, today) return events if as_list else len(events) - -def get_unseen_likes(): - """Returns count of unseen likes""" - - comment_doctype = DocType("Comment") - return frappe.db.count(comment_doctype, - filters=( - (comment_doctype.comment_type == "Like") - & (comment_doctype.modified >= Now() - Interval(years=1)) - & (comment_doctype.owner.notnull()) - & (comment_doctype.owner != frappe.session.user) - & (comment_doctype.reference_owner == frappe.session.user) - & (comment_doctype.seen == 0) - ) - ) From caab903edb79fac0420656416cf39584c33a87e6 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Mon, 7 Feb 2022 15:49:59 +0530 Subject: [PATCH 45/68] fix: add back `in_test` override for `imap_folders` test Since a lot of tests still uses the old way i.e, use `in_test` and execute custome (non-production) code to get results. Kept the same code for now else as a major test refactor needs to be done. --- .../doctype/email_account/email_account.py | 12 ++------ .../email_account/test_email_account.py | 29 ++++++++++++------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 424690e797..feef9dc234 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -457,7 +457,7 @@ class EmailAccount(Document): if exceptions: raise Exception(frappe.as_json(exceptions)) - def get_inbound_mails(self, test_mails=None, messages=None) -> List[InboundMail]: + def get_inbound_mails(self, test_mails=None) -> List[InboundMail]: """retrive and return inbound mails. """ @@ -471,14 +471,8 @@ class EmailAccount(Document): # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' mails.append(InboundMail(message, self, uid, seen_status, append_to)) - # if frappe.local.flags.in_test: - # if self.enable_incoming and not test_mails and messages: - # for folder in self.imap_folder: - # _messages = messages[folder.folder_name] if folder.folder_name in messages and (messages[folder.folder_name] is not None) else {} - # process_mail(_messages, folder.append_to) - # return mails - # else: - # return [InboundMail(msg, self) for msg in test_mails or []] + if frappe.local.flags.in_test: + return [InboundMail(msg, self) for msg in test_mails or []] if not self.enable_incoming: return [] diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 4c1acf091b..c53f8fa1ab 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -257,10 +257,7 @@ class TestEmailAccount(unittest.TestCase): self.assertTrue(communication.reference_name) self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) - @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) - @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) - # @patch("frappe.email.receive.EmailServer.get_messages", side_effect=get_mocked_messages) - def test_append_to_with_imap_folders(self, mocked_logout, mocked_select_imap_folder): + def test_append_to_with_imap_folders(self): mail_content_1 = self.get_test_mail(fname="incoming-1.raw") mail_content_2 = self.get_test_mail(fname="incoming-2.raw") mail_content_3 = self.get_test_mail(fname="incoming-3.raw") @@ -287,14 +284,10 @@ class TestEmailAccount(unittest.TestCase): 'uid_list': [2] } } - from frappe.email.receive import EmailServer - def get_mocked_messages(**args): - return messages[args["folder"]] - with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - mails = email_account.get_inbound_mails(messages=messages) - self.assertEqual(len(mails), 3) + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages) + self.assertEqual(len(mails), 3) inbox_mails = 0 test_folder_mails = 0 @@ -313,6 +306,20 @@ class TestEmailAccount(unittest.TestCase): self.assertEqual(inbox_mails, 2) self.assertEqual(test_folder_mails, 1) + @patch("frappe.local.flags.in_test", False) + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) + @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) + def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + from frappe.email.receive import EmailServer + + def get_mocked_messages(**kwargs): + return messages[kwargs["folder"]] + + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): + mails = email_account.get_inbound_mails() + + return mails + class TestInboundMail(unittest.TestCase): @classmethod def setUpClass(cls): From 266e1f95ed80811a816cad4816f8239eed08ca7e Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 7 Feb 2022 15:33:19 +0530 Subject: [PATCH 46/68] chore: adding back idx column This is needed for ordering in display of links as it keeps track of the clicks on a particular link item --- frappe/contacts/doctype/address/address.py | 2 +- frappe/contacts/doctype/contact/contact.py | 2 +- frappe/core/doctype/doctype/doctype.py | 1 - frappe/core/doctype/user_type/user_type.py | 2 +- frappe/database/database.py | 4 ++-- frappe/database/mariadb/framework_mariadb.sql | 1 + frappe/database/mariadb/schema.py | 4 ++-- frappe/database/postgres/framework_postgres.sql | 1 + frappe/database/postgres/schema.py | 4 ++-- frappe/database/schema.py | 2 +- frappe/desk/search.py | 2 +- frappe/model/__init__.py | 6 +++--- frappe/model/base_document.py | 10 +++++----- frappe/model/delete_doc.py | 1 - frappe/model/mapper.py | 2 +- frappe/public/js/frappe/model/model.js | 3 ++- frappe/tests/test_db_update.py | 4 ++-- frappe/website/website_generator.py | 1 + 18 files changed, 27 insertions(+), 25 deletions(-) diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 921990f2a3..5d0ed18d5f 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -241,7 +241,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): {mcond} {condition} order by if(locate(%(_txt)s, `tabAddress`.name), locate(%(_txt)s, `tabAddress`.name), 99999), - `tabAddress`.name + `tabAddress`.idx desc, `tabAddress`.name limit %(start)s, %(page_len)s """.format( mcond=get_match_cond(doctype), key=searchfield, diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index d3d0fba62c..9152655b85 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -215,7 +215,7 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters): {mcond} order by if(locate(%(_txt)s, `tabContact`.name), locate(%(_txt)s, `tabContact`.name), 99999), - `tabContact`.name + `tabContact`.idx desc, `tabContact`.name limit %(start)s, %(page_len)s """.format(mcond=get_match_cond(doctype), key=searchfield), { 'txt': '%' + txt + '%', '_txt': txt.replace("%", ""), diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 2df7516d93..d259367a16 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -707,7 +707,6 @@ class DocType(Document): add_column(self.name, "parent", "Data") add_column(self.name, "parenttype", "Data") add_column(self.name, "parentfield", "Data") - add_column(self.name, "idx", "Int", length=8, not_null=True, default="0") def get_max_idx(self): """Returns the highest `idx`""" diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index a64ad1a130..c0dfd2e597 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters ['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]] doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters, - limit_start=start, limit_page_length=page_len, as_list=1) + order_by='`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1) custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)], ['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']] diff --git a/frappe/database/database.py b/frappe/database/database.py index 3cebe4510d..9fa1ff161c 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -38,8 +38,8 @@ class Database(object): OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"] DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"] STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by') - DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus'] - CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield', 'idx') + DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'idx'] + CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield') MAX_WRITES_PER_TRANSACTION = 200_000 class InvalidColumnName(frappe.ValidationError): pass diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index b59beefa0b..7c9309ee9f 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -171,6 +171,7 @@ CREATE TABLE `tabDocType` ( `modified_by` varchar(255) DEFAULT NULL, `owner` varchar(255) DEFAULT NULL, `docstatus` int(1) NOT NULL DEFAULT 0, + `idx` int(8) NOT NULL DEFAULT 0, `search_fields` varchar(255) DEFAULT NULL, `issingle` int(1) NOT NULL DEFAULT 0, `is_tree` int(1) NOT NULL DEFAULT 0, diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 2f375bc751..0dc5863aa4 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -24,8 +24,7 @@ class MariaDBTable(DBTable): ( f"parent varchar({varchar_len})", f"parentfield varchar({varchar_len})", - f"parenttype varchar({varchar_len})", - "idx int(8) not null default '0'" + f"parenttype varchar({varchar_len})" ) ) + ',\n' @@ -37,6 +36,7 @@ class MariaDBTable(DBTable): modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus int(1) not null default '0', + idx int(8) not null default '0', {additional_definitions} index modified(modified)) ENGINE={engine} diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index 19381607d4..1662b7b93e 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -176,6 +176,7 @@ CREATE TABLE "tabDocType" ( "modified_by" varchar(255) DEFAULT NULL, "owner" varchar(255) DEFAULT NULL, "docstatus" smallint NOT NULL DEFAULT 0, + "idx" bigint NOT NULL DEFAULT 0, "search_fields" varchar(255) DEFAULT NULL, "issingle" smallint NOT NULL DEFAULT 0, "is_tree" smallint NOT NULL DEFAULT 0, diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index 9dbb597adf..9487bc2fa7 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -21,8 +21,7 @@ class PostgresTable(DBTable): ( "parent varchar({varchar_len})", "parentfield varchar({varchar_len})", - "parenttype varchar({varchar_len})", - "idx bigint not null default '0'" + "parenttype varchar({varchar_len})" ) ) @@ -35,6 +34,7 @@ class PostgresTable(DBTable): modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus smallint not null default '0', + idx bigint not null default '0', %s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN)) self.create_indexes() diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 7315d3630a..dd54385c83 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -108,7 +108,7 @@ class DBTable: frappe.db.STANDARD_VARCHAR_COLUMNS] if self.meta.get("istable"): columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in - frappe.db.CHILD_TABLE_COLUMNS if f != "idx"] + frappe.db.CHILD_TABLE_COLUMNS] columns += self.columns.values() for col in columns: diff --git a/frappe/desk/search.py b/frappe/desk/search.py index e89ef2be24..95397070ae 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -147,7 +147,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, from frappe.model.db_query import get_order_by order_by_based_on_meta = get_order_by(doctype, meta) # 2 is the index of _relevance column - order_by = "_relevance, {0}".format(order_by_based_on_meta) + order_by = "_relevance, {0}, `tab{1}`.idx desc".format(order_by_based_on_meta, doctype) ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read' ignore_permissions = True if doctype == "DocType" else (cint(ignore_user_permissions) and has_permission(doctype, ptype=ptype)) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index d635b31900..be9496c85b 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -90,14 +90,14 @@ default_fields = ( 'creation', 'modified', 'modified_by', - 'docstatus' + 'docstatus', + 'idx' ) child_table_fields = ( 'parent', 'parentfield', - 'parenttype', - 'idx' + 'parenttype' ) optional_fields = ( diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index bd3875cbea..307d95e84b 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -292,16 +292,16 @@ class BaseDocument(object): if key not in self.__dict__: self.__dict__[key] = None - if key == "docstatus" and self.__dict__[key] is None: - self.__dict__[key] = DocStatus.draft() + if self.__dict__[key] is None: + if key == "docstatus": + self.docstatus = DocStatus.draft() + elif key == "idx": + self.__dict__[key] = 0 for key in self.get_valid_columns(): if key not in self.__dict__: self.__dict__[key] = None - if key == "idx" and self.__dict__[key] is None: - self.__dict__[key] = 0 - def get_valid_columns(self): if self.doctype not in frappe.local.valid_columns: if self.doctype in DOCTYPES_FOR_DOCTYPE: diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index a888e3f7f7..ef73a349cc 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -232,7 +232,6 @@ def check_if_doc_is_linked(doc, method="Delete"): if frappe.get_meta(link_dt).istable: fields.extend(["parent", "parenttype"]) - # NOTE: scenario: parent doc <-(linked to) child table doc <-(linked to) doc for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True): # available only in child table cases item_parent = getattr(item, "parent", None) diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index 1ebff84a08..f40a43bb73 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -184,7 +184,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent): target_doc.set(fmap[1], val) # map idx - if source_doc.get("idx"): + if source_doc.idx: target_doc.idx = source_doc.idx # add fetch diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index ef161fca3d..89e029ffb1 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -10,7 +10,7 @@ $.extend(frappe.model, { layout_fields: ['Section Break', 'Column Break', 'Tab Break', 'Fold'], std_fields_list: ['name', 'owner', 'creation', 'modified', 'modified_by', - '_user_tags', '_comments', '_assign', '_liked_by', 'docstatus'], + '_user_tags', '_comments', '_assign', '_liked_by', 'docstatus', 'idx'], core_doctypes_list: ['DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role', 'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form', @@ -19,6 +19,7 @@ $.extend(frappe.model, { std_fields: [ {fieldname:'name', fieldtype:'Link', label:__('ID')}, {fieldname:'owner', fieldtype:'Link', label:__('Created By'), options: 'User'}, + {fieldname:'idx', fieldtype:'Int', label:__('Index')}, {fieldname:'creation', fieldtype:'Date', label:__('Created On')}, {fieldname:'modified', fieldtype:'Date', label:__('Last Updated On')}, {fieldname:'modified_by', fieldtype:'Data', label:__('Last Updated By')}, diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index 4c0cc4b2bc..66eb05391a 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -106,6 +106,7 @@ def get_other_fields_meta(meta): 'modified_by': ('Data', 0), 'creation': ('Datetime', 0), 'modified': ('Datetime', 0), + 'idx': ('Int', 8), 'docstatus': ('Check', 0) } @@ -115,8 +116,7 @@ def get_other_fields_meta(meta): child_table_fields_map = {} if meta.istable: - child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS if field != 'idx'}) - child_table_fields_map.update({'idx': ('Int', 8)}) + child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS}) optional_fields_map = {field: ('Text', 0) for field in optional_fields} fields = dict(default_fields_map, **optional_fields_map, **child_table_fields_map) diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py index 67b8e77c4f..e66496aa89 100644 --- a/frappe/website/website_generator.py +++ b/frappe/website/website_generator.py @@ -118,6 +118,7 @@ class WebsiteGenerator(Document): "doc": self, "page_or_generator": "Generator", "ref_doctype":self.doctype, + "idx": self.idx, "docname": self.name, "controller": get_module_name(self.doctype, self.meta.module), }) From 7bd84496c1d913a40a912643a1f82c5e612e9d89 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 7 Feb 2022 16:42:39 +0530 Subject: [PATCH 47/68] fix: use int for docstatus in get_default_df --- frappe/model/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index ef8eca3629..372392f689 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -678,7 +678,7 @@ def get_default_df(fieldname): fieldtype = "Datetime" ) - elif fieldname == "idx": + elif fieldname in ("idx", "docstatus"): return frappe._dict( fieldname = fieldname, fieldtype = "Int" From a6ea0c9e37e2c5099ad60252f78a11978c09bae6 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 7 Feb 2022 17:49:37 +0530 Subject: [PATCH 48/68] fix: add index on parent in child table doctype --- frappe/database/mariadb/schema.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 0dc5863aa4..fd4bfc6dd0 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -24,7 +24,8 @@ class MariaDBTable(DBTable): ( f"parent varchar({varchar_len})", f"parentfield varchar({varchar_len})", - f"parenttype varchar({varchar_len})" + f"parenttype varchar({varchar_len})", + "index parent(parent)" ) ) + ',\n' From 7c51c6a9ea1866ab9e19ed8a60fef235fa716782 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Mon, 7 Feb 2022 18:25:36 +0530 Subject: [PATCH 49/68] fix: add in_test parameter from frappe.local.flags.in_test mock --- frappe/email/doctype/email_account/test_email_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index c53f8fa1ab..46cc3d03af 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -309,7 +309,7 @@ class TestEmailAccount(unittest.TestCase): @patch("frappe.local.flags.in_test", False) @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) - def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None, mocked_in_test=None): from frappe.email.receive import EmailServer def get_mocked_messages(**kwargs): From c5fbd0784048d300157dd64c9149be535ed46953 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 7 Feb 2022 19:10:42 +0530 Subject: [PATCH 50/68] revert: "fix: use `now` instead of `is_async` in tests (#15893)" (#15899) This reverts commit 1cd47222353accf9bcbf403182fb52fc84bdbf7e. --- frappe/model/delete_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index ef73a349cc..2cc99575d6 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -115,7 +115,7 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa # All the linked docs should be checked beforehand frappe.enqueue('frappe.model.delete_doc.delete_dynamic_links', doctype=doc.doctype, name=doc.name, - now=frappe.flags.in_test) + is_async=False if frappe.flags.in_test else True) # clear cache for Document doc.clear_cache() From 05e29fa496dc15244be6b30464c06890465208a3 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 7 Feb 2022 21:15:41 +0530 Subject: [PATCH 51/68] test: ignore dataimport test on postgres (#15902) Co-authored-by: Deepesh Garg --- frappe/core/doctype/data_import/test_importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index 3f78594dd2..11077ca58b 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -4,6 +4,7 @@ import unittest import frappe from frappe.core.doctype.data_import.importer import Importer +from frappe.tests.test_query_builder import db_type_is, run_only_if from frappe.utils import getdate, format_duration doctype_name = 'DocType for Import' @@ -54,6 +55,8 @@ class TestImporter(unittest.TestCase): self.assertEqual(len(preview.data), 4) self.assertEqual(len(preview.columns), 16) + # ignored on postgres because myisam doesn't exist on pg + @run_only_if(db_type_is.MARIADB) def test_data_import_without_mandatory_values(self): import_file = get_import_file('sample_import_file_without_mandatory') data_import = self.get_importer(doctype_name, import_file) From 7b0e646e10445c5bca6dcb5c0848a0045581e06e Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 8 Feb 2022 01:04:44 +0530 Subject: [PATCH 52/68] fix: none type issue in email test, add more coverage to `test_auto_reply` --- .../email_account/test_email_account.py | 26 ++++++++++++++-- .../email/doctype/email_queue/email_queue.py | 31 ++++++++++--------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 46cc3d03af..2c5f69333b 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -213,10 +213,20 @@ class TestEmailAccount(unittest.TestCase): def test_auto_reply(self): cleanup("test_sender@example.com") - test_mails = [self.get_test_mail('incoming-1.raw')] + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + self.get_test_mail('incoming-1.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, @@ -319,6 +329,18 @@ class TestEmailAccount(unittest.TestCase): mails = email_account.get_inbound_mails() return mails + + @patch("frappe.local.flags.in_test", False) + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) + @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) + def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None, mocked_in_test=None): + def get_mocked_messages(**kwargs): + return messages[kwargs["folder"]] + + from frappe.email.receive import EmailServer + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.receive() class TestInboundMail(unittest.TestCase): @classmethod diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 4da83bd0d2..9b4f3b984c 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -479,21 +479,24 @@ class QueueBuilder: EmailUnsubscribe = DocType("Email Unsubscribe") - unsubscribed = ( - frappe.qb.from_(EmailUnsubscribe).select( - EmailUnsubscribe.email - ).where( - EmailUnsubscribe.email.isin(all_ids) - & ( - ( - (EmailUnsubscribe.reference_doctype == self.reference_doctype) - & (EmailUnsubscribe.reference_name == self.reference_name) - ) | ( - EmailUnsubscribe.global_unsubscribe == 1 + if len(all_ids) > 0: + unsubscribed = ( + frappe.qb.from_(EmailUnsubscribe).select( + EmailUnsubscribe.email + ).where( + EmailUnsubscribe.email.isin(all_ids) + & ( + ( + (EmailUnsubscribe.reference_doctype == self.reference_doctype) + & (EmailUnsubscribe.reference_name == self.reference_name) + ) | ( + EmailUnsubscribe.global_unsubscribe == 1 + ) ) - ) - ).distinct() - ).run(pluck=True) + ).distinct() + ).run(pluck=True) + else: + unsubscribed = None self._unsubscribed_user_emails = unsubscribed or [] return self._unsubscribed_user_emails From d318675522e3fdfa3e525deae67076769965ed13 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 8 Feb 2022 01:27:26 +0530 Subject: [PATCH 53/68] feat: add coverage for some more email tests - test_incoming - test_incoming_with_attach - test_incoming_attached_email_from_outlook_plain_text_only - test_incoming_attached_email_from_outlook_layers --- .../email_account/test_email_account.py | 59 +++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 2c5f69333b..d9f47a6026 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -45,10 +45,20 @@ class TestEmailAccount(unittest.TestCase): def test_incoming(self): cleanup("test_sender@example.com") - test_mails = [self.get_test_mail('incoming-1.raw')] + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + self.get_test_mail('incoming-1.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) @@ -72,11 +82,20 @@ class TestEmailAccount(unittest.TestCase): existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile: - test_mails = [testfile.read()] + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + self.get_test_mail('incoming-2.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) @@ -93,11 +112,20 @@ class TestEmailAccount(unittest.TestCase): def test_incoming_attached_email_from_outlook_plain_text_only(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-3.raw"), "r") as f: - test_mails = [f.read()] + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + self.get_test_mail('incoming-3.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) @@ -106,11 +134,20 @@ class TestEmailAccount(unittest.TestCase): def test_incoming_attached_email_from_outlook_layers(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-4.raw"), "r") as f: - test_mails = [f.read()] + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + self.get_test_mail('incoming-4.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) From 0e63c9c33e8e92043a87e74c68e7eb874395120d Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 8 Feb 2022 08:11:43 +0530 Subject: [PATCH 54/68] fix: test fail issue for some email tests - test_threading - test_threading_by_message_id - test_threading_by_subject --- .../email_account/test_email_account.py | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index d9f47a6026..fc8b678fde 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -188,11 +188,22 @@ class TestEmailAccount(unittest.TestCase): with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw"), "r") as f: raw = f.read() raw = raw.replace("<-- in-reply-to -->", sent_mail.get("Message-Id")) - test_mails = [raw] # parse reply + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + raw + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) sent = frappe.get_doc("Communication", sent_name) @@ -210,8 +221,19 @@ class TestEmailAccount(unittest.TestCase): test_mails.append(f.read()) # parse reply + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': test_mails, + 'seen_status': { + 2: 'UNSEEN', + 3: 'UNSEEN' + }, + 'uid_list': [2, 3] + } + } + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) @@ -234,11 +256,22 @@ class TestEmailAccount(unittest.TestCase): # get test mail with message-id as in-reply-to with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f: - test_mails = [f.read().replace('{{ message_id }}', last_mail.message_id)] + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + f.read().replace('{{ message_id }}', last_mail.message_id) + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } # pull the mail + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) From 84501c663ea9c30ff3d354915e5df5f0fd4ff1e0 Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 8 Feb 2022 08:33:09 +0530 Subject: [PATCH 55/68] fix: use new mocked_inbound_mail in test_email - remove in_test block from production code --- .../doctype/email_account/email_account.py | 3 --- .../email_account/test_email_account.py | 6 ++--- frappe/tests/test_email.py | 26 ++++++++++++++++++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index feef9dc234..f20a25f88f 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -471,9 +471,6 @@ class EmailAccount(Document): # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' mails.append(InboundMail(message, self, uid, seen_status, append_to)) - if frappe.local.flags.in_test: - return [InboundMail(msg, self) for msg in test_mails or []] - if not self.enable_incoming: return [] diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index fc8b678fde..06c4323f4e 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -386,10 +386,9 @@ class TestEmailAccount(unittest.TestCase): self.assertEqual(inbox_mails, 2) self.assertEqual(test_folder_mails, 1) - @patch("frappe.local.flags.in_test", False) @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) - def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None, mocked_in_test=None): + def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): from frappe.email.receive import EmailServer def get_mocked_messages(**kwargs): @@ -400,10 +399,9 @@ class TestEmailAccount(unittest.TestCase): return mails - @patch("frappe.local.flags.in_test", False) @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) - def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None, mocked_in_test=None): + def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): def get_mocked_messages(**kwargs): return messages[kwargs["folder"]] diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index ef9515f5ba..34c401b7b0 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -3,6 +3,8 @@ import unittest, frappe, re, email +from frappe.email.doctype.email_account.test_email_account import TestEmailAccount + test_dependencies = ['Email Account'] class TestEmail(unittest.TestCase): @@ -173,12 +175,34 @@ class TestEmail(unittest.TestCase): frappe.db.delete("Communication", {"sender": "sukh@yyy.com"}) with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw: - mails = email_account.get_inbound_mails(test_mails=[raw.read()]) + messages = { + '"INBOX"': { # append_to = ToDo + 'latest_messages': [ + raw.read() + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + changed_flag = False + if not email_account.enable_incoming: + email_account.enable_incoming = True + changed_flag = True + mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages) + + # mails = email_account.get_inbound_mails(test_mails=[raw.read()]) communication = mails[0].process() self.assertTrue(re.search(''']*src=["']/private/files/rtco1.png[^>]*>''', communication.content)) self.assertTrue(re.search(''']*src=["']/private/files/rtco2.png[^>]*>''', communication.content)) + if changed_flag: + email_account.enable_incoming = False + if __name__ == '__main__': frappe.connect() From b978b5a70be3b7c091602fc558fe9588ba3c87ad Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 8 Feb 2022 12:01:11 +0530 Subject: [PATCH 56/68] test: Fix mocked method and formatting issues --- .../email_account/test_email_account.py | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 06c4323f4e..c0ce177f4b 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -46,7 +46,8 @@ class TestEmailAccount(unittest.TestCase): cleanup("test_sender@example.com") messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ self.get_test_mail('incoming-1.raw') ], @@ -54,7 +55,7 @@ class TestEmailAccount(unittest.TestCase): 2: 'UNSEEN' }, 'uid_list': [2] - } + } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -83,7 +84,8 @@ class TestEmailAccount(unittest.TestCase): frappe.delete_doc("File", existing_file.name) messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ self.get_test_mail('incoming-2.raw') ], @@ -91,7 +93,7 @@ class TestEmailAccount(unittest.TestCase): 2: 'UNSEEN' }, 'uid_list': [2] - } + } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -113,7 +115,8 @@ class TestEmailAccount(unittest.TestCase): cleanup("test_sender@example.com") messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ self.get_test_mail('incoming-3.raw') ], @@ -121,7 +124,7 @@ class TestEmailAccount(unittest.TestCase): 2: 'UNSEEN' }, 'uid_list': [2] - } + } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -135,7 +138,8 @@ class TestEmailAccount(unittest.TestCase): cleanup("test_sender@example.com") messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ self.get_test_mail('incoming-4.raw') ], @@ -143,7 +147,7 @@ class TestEmailAccount(unittest.TestCase): 2: 'UNSEEN' }, 'uid_list': [2] - } + } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -199,7 +203,7 @@ class TestEmailAccount(unittest.TestCase): 2: 'UNSEEN' }, 'uid_list': [2] - } + } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -229,7 +233,7 @@ class TestEmailAccount(unittest.TestCase): 3: 'UNSEEN' }, 'uid_list': [2, 3] - } + } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -257,7 +261,8 @@ class TestEmailAccount(unittest.TestCase): # get test mail with message-id as in-reply-to with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f: messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ f.read().replace('{{ message_id }}', last_mail.message_id) ], @@ -265,11 +270,10 @@ class TestEmailAccount(unittest.TestCase): 2: 'UNSEEN' }, 'uid_list': [2] - } + } } # pull the mail - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") TestEmailAccount.mocked_email_receive(email_account, messages) @@ -284,7 +288,8 @@ class TestEmailAccount(unittest.TestCase): cleanup("test_sender@example.com") messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ self.get_test_mail('incoming-1.raw') ], @@ -292,7 +297,7 @@ class TestEmailAccount(unittest.TestCase): 2: 'UNSEEN' }, 'uid_list': [2] - } + } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -343,7 +348,8 @@ class TestEmailAccount(unittest.TestCase): mail_content_3 = self.get_test_mail(fname="incoming-3.raw") messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ mail_content_1, mail_content_2 @@ -354,7 +360,8 @@ class TestEmailAccount(unittest.TestCase): }, 'uid_list': [0,1] }, - '"Test Folder"': { # append_to = Communication + # append_to = Communication + '"Test Folder"': { 'latest_messages': [ mail_content_3 ], @@ -392,22 +399,21 @@ class TestEmailAccount(unittest.TestCase): from frappe.email.receive import EmailServer def get_mocked_messages(**kwargs): - return messages[kwargs["folder"]] + return messages.get(kwargs["folder"], {}) with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): mails = email_account.get_inbound_mails() return mails - + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): def get_mocked_messages(**kwargs): - return messages[kwargs["folder"]] + return messages.get(kwargs["folder"], {}) from frappe.email.receive import EmailServer with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") email_account.receive() class TestInboundMail(unittest.TestCase): From cb25d894b4c6804a2cbd074ed88267eab2aa0dcc Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Tue, 8 Feb 2022 12:54:51 +0530 Subject: [PATCH 57/68] refactor: code cleanup --- frappe/email/doctype/email_account/email_account.py | 10 +++++----- .../email/doctype/email_account/test_email_account.py | 6 ++++-- frappe/tests/test_email.py | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index f20a25f88f..3a1b683398 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -421,10 +421,10 @@ class EmailAccount(Document): def get_failed_attempts_count(self): return cint(frappe.cache().get('{0}:email-account-failed-attempts'.format(self.name))) - def receive(self, test_mails=None): + def receive(self): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" exceptions = [] - inbound_mails = self.get_inbound_mails(test_mails=test_mails) + inbound_mails = self.get_inbound_mails() for mail in inbound_mails: try: communication = mail.process() @@ -457,7 +457,7 @@ class EmailAccount(Document): if exceptions: raise Exception(frappe.as_json(exceptions)) - def get_inbound_mails(self, test_mails=None) -> List[InboundMail]: + def get_inbound_mails(self) -> List[InboundMail]: """retrive and return inbound mails. """ @@ -466,8 +466,8 @@ class EmailAccount(Document): def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages['uid_list'][index] if messages.get('uid_list') else None - seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0 - if not (self.email_sync_option == 'UNSEEN' and seen_status): + seen_status = messages.get('seen_status', {}).get(uid) + if self.email_sync_option != 'UNSEEN' or seen_status != "SEEN": # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' mails.append(InboundMail(message, self, uid, seen_status, append_to)) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index c0ce177f4b..f609c2947d 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -195,7 +195,8 @@ class TestEmailAccount(unittest.TestCase): # parse reply messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ raw ], @@ -226,7 +227,8 @@ class TestEmailAccount(unittest.TestCase): # parse reply messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': test_mails, 'seen_status': { 2: 'UNSEEN', diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index 34c401b7b0..ad9f8fdd11 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -176,7 +176,8 @@ class TestEmail(unittest.TestCase): with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw: messages = { - '"INBOX"': { # append_to = ToDo + # append_to = ToDo + '"INBOX"': { 'latest_messages': [ raw.read() ], From 496e5b513bc3f67cb4915d9c201e9c15d54c7da1 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 8 Feb 2022 13:02:47 +0530 Subject: [PATCH 58/68] fix: executing non-select qb code from whitelisted methods Co-Authored-By: Gavin D'souza --- frappe/query_builder/utils.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index cbd6147e01..1ddf4fc034 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -59,10 +59,28 @@ def patch_query_execute(): return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep def prepare_query(query): + import inspect + param_collector = NamedParameterWrapper() query = query.get_sql(param_wrapper=param_collector) if frappe.flags.in_safe_exec and not query.lower().strip().startswith("select"): - raise frappe.PermissionError('Only SELECT SQL allowed in scripting') + callstack = inspect.stack() + if len(callstack) >= 3 and ".py" in callstack[2].filename: + # ignore any query builder methods called from python files + # assumption is that those functions are whitelisted already. + + # since query objects are patched everywhere any query.run() + # will have callstack like this: + # frame0: this function prepare_query() + # frame1: execute_query() + # frame2: frame that called `query.run()` + # + # if frame2 is server script it wont have a filename and hence + # it shouldn't be allowed. + # ps. stack() returns `""` as filename. + pass + else: + raise frappe.PermissionError('Only SELECT SQL allowed in scripting') return query, param_collector.get_parameters() query_class = get_attr(str(frappe.qb).split("'")[1]) From b64c03d83d2e265b142cc4f4503345348c81128f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 8 Feb 2022 13:20:25 +0530 Subject: [PATCH 59/68] test: restricted QB --- .../server_script/test_server_script.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index bc92061f42..d9381bcd16 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -139,3 +139,42 @@ class TestServerScript(unittest.TestCase): server_script.disabled = 1 server_script.save() + + def test_restricted_qb(self): + todo = frappe.get_doc(doctype="ToDo", description="QbScriptTestNote") + todo.insert() + + script = frappe.get_doc( + doctype='Server Script', + name='test_qb_restrictions', + script_type = 'API', + api_method = 'test_qb_restrictions', + allow_guest = 1, + # whitelisted update + script = f''' +frappe.db.set_value("ToDo", "{todo.name}", "description", "safe") +''' + ) + script.insert() + script.execute_method() + + todo.reload() + self.assertEqual(todo.description, "safe") + + # unsafe update + script.script = f""" +todo = frappe.qb.DocType("ToDo") +frappe.qb.update(todo).set(todo.description, "unsafe").where(todo.name == "{todo.name}").run() +""" + script.save() + self.assertRaises(frappe.PermissionError, script.execute_method) + todo.reload() + self.assertEqual(todo.description, "safe") + + # safe select + script.script = f""" +todo = frappe.qb.DocType("ToDo") +frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run() +""" + script.save() + script.execute_method() From 2b12fa880c3900c25f8e8dda5384233714e2e63b Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 8 Feb 2022 17:02:48 +0530 Subject: [PATCH 60/68] fix: UI fix for card block --- frappe/public/js/frappe/widgets/base_widget.js | 2 +- frappe/public/scss/desk/desktop.scss | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/widgets/base_widget.js b/frappe/public/js/frappe/widgets/base_widget.js index aabb3526b0..dbcacbf50e 100644 --- a/frappe/public/js/frappe/widgets/base_widget.js +++ b/frappe/public/js/frappe/widgets/base_widget.js @@ -100,7 +100,7 @@ export default class Widget { let title = max_chars ? frappe.ellipsis(base, max_chars) : base; if (this.icon) { - let icon = frappe.utils.icon(this.icon); + let icon = frappe.utils.icon(this.icon, "md"); this.title_field[0].innerHTML = `${icon} ${title}`; } else { this.title_field[0].innerHTML = `${title}`; diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss index b471feb878..435dafde87 100644 --- a/frappe/public/scss/desk/desktop.scss +++ b/frappe/public/scss/desk/desktop.scss @@ -154,7 +154,7 @@ body { svg { flex: none; - margin-right: 6px; + margin-right: var(--margin-sm); box-shadow: none; } } @@ -560,6 +560,8 @@ body { } &.links-widget-box { + padding: 16px 7px; + .link-item { display: flex; text-decoration: none; @@ -601,6 +603,8 @@ body { .indicator-pill { margin-right: var(--margin-sm); + height: 20px; + padding: 3px 8px; } } } From c12c8eb31444476ae6831b5709db5c23f6e7fef3 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 8 Feb 2022 19:28:08 +0530 Subject: [PATCH 61/68] style: made links card block pixel perfect --- .../public/js/frappe/widgets/base_widget.js | 2 +- frappe/public/scss/desk/desktop.scss | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/frappe/public/js/frappe/widgets/base_widget.js b/frappe/public/js/frappe/widgets/base_widget.js index dbcacbf50e..45d4926904 100644 --- a/frappe/public/js/frappe/widgets/base_widget.js +++ b/frappe/public/js/frappe/widgets/base_widget.js @@ -100,7 +100,7 @@ export default class Widget { let title = max_chars ? frappe.ellipsis(base, max_chars) : base; if (this.icon) { - let icon = frappe.utils.icon(this.icon, "md"); + let icon = frappe.utils.icon(this.icon, "lg"); this.title_field[0].innerHTML = `${icon} ${title}`; } else { this.title_field[0].innerHTML = `${title}`; diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss index 435dafde87..fd9ee05f27 100644 --- a/frappe/public/scss/desk/desktop.scss +++ b/frappe/public/scss/desk/desktop.scss @@ -154,7 +154,8 @@ body { svg { flex: none; - margin-right: var(--margin-sm); + margin-right: 6px; + margin-left: -2px; box-shadow: none; } } @@ -560,23 +561,29 @@ body { } &.links-widget-box { - padding: 16px 7px; + padding: 18px 12px; .link-item { display: flex; text-decoration: none; + font-size: var(--text-md); color: var(--text-color); - padding: var(--padding-xs); - margin-left: -5px; + padding: 4px; + margin-left: -4px; + margin-bottom: 4px; border-radius: var(--border-radius-md); cursor: pointer; &:hover { - background-color: var(--bg-color); + background-color: var(--fg-hover-color); + + .indicator-pill { + background-color: var(--fg-color); + } } &:first-child { - margin-top: 15px; + margin-top: 18px; } &:last-child { From 951ad3f8441c3129259a05e59a1e9c0aeac6e0ea Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sun, 6 Feb 2022 02:17:35 +0530 Subject: [PATCH 62/68] chore(deps): bump werkzeug to latest version --- frappe/app.py | 21 ---------- frappe/desk/page/setup_wizard/setup_wizard.py | 42 ++++++++++++------- requirements.txt | 3 +- 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/frappe/app.py b/frappe/app.py index 609a8535d7..975a2e2002 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -294,7 +294,6 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No _sites_path = sites_path from werkzeug.serving import run_simple - patch_werkzeug_reloader() if profile or os.environ.get('USE_PROFILER'): application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls')) @@ -325,23 +324,3 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No use_debugger=not in_test_env, use_evalex=not in_test_env, threaded=not no_threading) - -def patch_werkzeug_reloader(): - """ - This function monkey patches Werkzeug reloader to ignore reloading files in - the __pycache__ directory. - - To be deprecated when upgrading to Werkzeug 2. - """ - - from werkzeug._reloader import WatchdogReloaderLoop - - trigger_reload = WatchdogReloaderLoop.trigger_reload - - def custom_trigger_reload(self, filename): - if os.path.basename(os.path.dirname(filename)) == "__pycache__": - return - - return trigger_reload(self, filename) - - WatchdogReloaderLoop.trigger_reload = custom_trigger_reload diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index b42d8c58b7..8dc010e420 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -6,7 +6,7 @@ from frappe.utils import strip, cint from frappe.translate import (set_default_language, get_dict, send_translations) from frappe.geo.country_info import get_country_info from frappe.utils.password import update_password -from werkzeug.useragents import UserAgent +from ua_parser import user_agent_parser from . import install_fixtures def get_setup_stages(args): @@ -315,17 +315,10 @@ def prettify_args(args): return pretty_args def email_setup_wizard_exception(traceback, args): - if not frappe.local.conf.setup_wizard_exception_email: + if not frappe.conf.setup_wizard_exception_email: return pretty_args = prettify_args(args) - - if frappe.local.request: - user_agent = UserAgent(frappe.local.request.headers.get('User-Agent', '')) - - else: - user_agent = frappe._dict() - message = """ #### Traceback @@ -350,22 +343,43 @@ def email_setup_wizard_exception(traceback, args): - **Site:** {site} - **User:** {user} -- **Browser:** {user_agent.platform} {user_agent.browser} version: {user_agent.version} language: {user_agent.language} +- **Browser:** {browser} - **Browser Languages**: `{accept_languages}`""".format( site=frappe.local.site, traceback=traceback, args="\n".join(pretty_args), user=frappe.session.user, - user_agent=user_agent, - headers=frappe.local.request.headers, - accept_languages=", ".join(frappe.local.request.accept_languages.values())) + browser=get_browser_string(), + headers=frappe.request.headers, + accept_languages=", ".join(frappe.request.accept_languages.values())) - frappe.sendmail(recipients=frappe.local.conf.setup_wizard_exception_email, + frappe.sendmail(recipients=frappe.conf.setup_wizard_exception_email, sender=frappe.session.user, subject="Setup failed: {}".format(frappe.local.site), message=message, delayed=False) +def get_browser_string(): + if not frappe.request: + return "" + + user_agent = user_agent_parser.ParseUserAgent( + frappe.request.headers.get('User-Agent', '') + ) + + browser = user_agent["family"] + + if user_agent["major"]: + browser += " " + user_agent["major"] + + if user_agent["minor"]: + browser += "." + user_agent["minor"] + + if user_agent["patch"]: + browser += "." + user_agent["patch"] + + return browser + def log_setup_wizard_exception(traceback, args): with open('../logs/setup-wizard.log', 'w+') as setup_log: setup_log.write(traceback) diff --git a/requirements.txt b/requirements.txt index 114ab5f61d..4426ef54be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -62,8 +62,9 @@ semantic-version~=2.8.5 sqlparse~=0.4.1 stripe~=2.56.0 terminaltables~=3.1.0 +ua-parser~=0.10.0 urllib3~=1.26.4 -Werkzeug~=0.16.1 +Werkzeug~=2.0.2 Whoosh~=2.7.4 wrapt~=1.12.1 xlrd~=2.0.1 From e5857fa56ad651217e2300b991704f6fc8b1bd6c Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 8 Feb 2022 19:58:56 +0530 Subject: [PATCH 63/68] fix: improve setup wizard exception email --- frappe/desk/page/setup_wizard/setup_wizard.py | 29 ++----------------- requirements.txt | 3 +- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index 8dc010e420..0c32e886f4 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -6,7 +6,6 @@ from frappe.utils import strip, cint from frappe.translate import (set_default_language, get_dict, send_translations) from frappe.geo.country_info import get_country_info from frappe.utils.password import update_password -from ua_parser import user_agent_parser from . import install_fixtures def get_setup_stages(args): @@ -342,16 +341,13 @@ def email_setup_wizard_exception(traceback, args): #### Basic Information - **Site:** {site} -- **User:** {user} -- **Browser:** {browser} -- **Browser Languages**: `{accept_languages}`""".format( +- **User:** {user}""".format( site=frappe.local.site, traceback=traceback, args="\n".join(pretty_args), user=frappe.session.user, - browser=get_browser_string(), headers=frappe.request.headers, - accept_languages=", ".join(frappe.request.accept_languages.values())) + ) frappe.sendmail(recipients=frappe.conf.setup_wizard_exception_email, sender=frappe.session.user, @@ -359,27 +355,6 @@ def email_setup_wizard_exception(traceback, args): message=message, delayed=False) -def get_browser_string(): - if not frappe.request: - return "" - - user_agent = user_agent_parser.ParseUserAgent( - frappe.request.headers.get('User-Agent', '') - ) - - browser = user_agent["family"] - - if user_agent["major"]: - browser += " " + user_agent["major"] - - if user_agent["minor"]: - browser += "." + user_agent["minor"] - - if user_agent["patch"]: - browser += "." + user_agent["patch"] - - return browser - def log_setup_wizard_exception(traceback, args): with open('../logs/setup-wizard.log', 'w+') as setup_log: setup_log.write(traceback) diff --git a/requirements.txt b/requirements.txt index 4426ef54be..2e05159203 100644 --- a/requirements.txt +++ b/requirements.txt @@ -62,9 +62,8 @@ semantic-version~=2.8.5 sqlparse~=0.4.1 stripe~=2.56.0 terminaltables~=3.1.0 -ua-parser~=0.10.0 urllib3~=1.26.4 -Werkzeug~=2.0.2 +Werkzeug~=2.0.3 Whoosh~=2.7.4 wrapt~=1.12.1 xlrd~=2.0.1 From 240665940f52dd0485847b99a099ad3d857d44ff Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Wed, 9 Feb 2022 11:01:05 +0530 Subject: [PATCH 64/68] fix: remove redundant test record deletions and resetting --- frappe/tests/test_naming.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index f1c7ad42d0..3e1120dc79 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -11,15 +11,9 @@ from frappe.model.naming import determine_consecutive_week_number, parse_naming_ class TestNaming(unittest.TestCase): def setUp(self): - frappe.db.sql('delete from `tabNote`') - frappe.db.sql('delete from `tabToDo`') + frappe.db.delete('Note') def tearDown(self): - # Reset ToDo autoname to hash - todo_doctype = frappe.get_doc('DocType', 'ToDo') - todo_doctype.autoname = 'hash' - todo_doctype.save() - frappe.db.rollback() def test_append_number_if_name_exists(self): From 9b9956e1ce67f3a511921c2e28d0618d8b84d1e7 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Wed, 9 Feb 2022 12:15:10 +0530 Subject: [PATCH 65/68] fix: blur event was overlapping block list item click event --- .../public/js/frappe/views/workspace/blocks/paragraph.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/workspace/blocks/paragraph.js b/frappe/public/js/frappe/views/workspace/blocks/paragraph.js index 70f97c44c1..99c161439f 100644 --- a/frappe/public/js/frappe/views/workspace/blocks/paragraph.js +++ b/frappe/public/js/frappe/views/workspace/blocks/paragraph.js @@ -62,7 +62,7 @@ export default class Paragraph extends Block { this.show_hide_block_list(); }); div.addEventListener('blur', () => { - setTimeout(() => this.show_hide_block_list(true), 10); + !this.over_block_list_item && this.show_hide_block_list(true); }); div.dataset.placeholder = this.api.i18n.t(this._placeholder); div.addEventListener('keyup', this.onKeyUp); @@ -95,6 +95,12 @@ export default class Paragraph extends Block { this.api.caret.setToBlock(index); }); + $block_list_item.mouseenter(() => { + this.over_block_list_item = true; + }).mouseleave(() => { + this.over_block_list_item = false; + }); + $block_list_container.append($block_list_item); }); From 5798cfaf4c76ac3feee033bdd6f194d6ff31f557 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 9 Feb 2022 12:12:24 +0530 Subject: [PATCH 66/68] build: Update iPython dependency Updating dependency due to arbitrary code execution vulnerability in IPython that stems from IPython executing untrusted files in CWD. This vulnerability allows one user to run code as another. ref: https://ipython.readthedocs.io/en/stable/whatsnew/version8.html#ipython-8-0-1-cve-2022-21699 Weaknesses: CWE-250, CWE-269, CWE-279 CVE ID: CVE-2022-21699 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 114ab5f61d..f47c296843 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ googlemaps~=4.4.5 gunicorn~=20.1.0 html2text==2020.1.16 html5lib~=1.1 -ipython~=7.27.0 +ipython~=7.31.1 Jinja2~=3.0.1 ldap3~=2.9 markdown2~=2.4.0 From adc69cb3ecd35d2bc34aab0d5e67d15a70820ae1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 9 Feb 2022 12:19:43 +0530 Subject: [PATCH 67/68] build: Upgrade Pillow dependency This upgrade handles multiple high severity vulnerabilities. I've not checked the affected code in great depth but the APIs we use may be affected. If they could actually be exploited is another matter which would take a whole lotta effort which I'd rather not test xD Fixes: CWE-74, CWE-125, CWE-120, CWE-125, CWE-400 CVE IDs: CVE-2022-22817, CVE-2022-22816, CVE-2021-34552, CVE-2021-23437 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f47c296843..97536d0e56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ openpyxl~=3.0.7 passlib~=1.7.4 paytmchecksum~=1.7.0 pdfkit~=0.6.1 -Pillow~=8.2.0 +Pillow~=9.0.0 premailer~=3.8.0 psutil~=5.8.0 psycopg2-binary~=2.9.1 From 05bb38ea46fce3f0bed2a317cf630b6fd5e77b0d Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Wed, 9 Feb 2022 12:32:26 +0530 Subject: [PATCH 68/68] chore: fetch name only if first_document is checked --- frappe/desk/doctype/form_tour/form_tour.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 19fe623f60..d6390d7613 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -15,12 +15,12 @@ frappe.ui.form.on('Form Tour', { frm.add_custom_button(__('Show Tour'), async () => { const issingle = await check_if_single(frm.doc.reference_doctype); - const name = await get_first_document(frm.doc.reference_doctype); let route_changed = null; - + if (issingle) { route_changed = frappe.set_route('Form', frm.doc.reference_doctype); } else if (frm.doc.first_document) { + const name = await get_first_document(frm.doc.reference_doctype); route_changed = frappe.set_route('Form', frm.doc.reference_doctype, name); } else { route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new');