diff --git a/cypress/integration/control_link.js b/cypress/integration/control_link.js index 6d16769b37..bfa70ad338 100644 --- a/cypress/integration/control_link.js +++ b/cypress/integration/control_link.js @@ -58,6 +58,23 @@ context('Control Link', () => { cy.get('.frappe-control[data-fieldname=link] input').should('have.value', ''); }); + it("should be possible set empty value explicitly", () => { + get_dialog_with_link().as("dialog"); + + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); + + cy.get(".frappe-control[data-fieldname=link] input") + .type(" ", { delay: 100 }) + .blur(); + cy.wait("@validate_link"); + cy.get(".frappe-control[data-fieldname=link] input").should("have.value", ""); + cy.window() + .its("cur_dialog") + .then((dialog) => { + expect(dialog.get_value("link")).to.equal(''); + }); + }); + it('should route to form on arrow click', () => { get_dialog_with_link().as('dialog'); @@ -78,7 +95,7 @@ context('Control Link', () => { }); }); - it('should fetch valid value', () => { + it('should update dependant fields (via fetch_from)', () => { cy.get('@todos').then(todos => { cy.visit(`/app/todo/${todos[0]}`); cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); @@ -89,7 +106,67 @@ context('Control Link', () => { cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( 'contain', 'Administrator' ); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", "Administrator"); + + // invalid input + cy.get('@input').clear().type('invalid input', {delay: 100}).blur(); + cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( + 'contain', '' + ); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", null); + + // set valid value again + cy.get('@input').clear().type('Administrator', {delay: 100}).blur(); + cy.wait('@validate_link'); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", "Administrator"); + + // clear input + cy.get('@input').clear().blur(); + cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( + 'contain', '' + ); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", ""); }); }); + it("should set default values", () => { + cy.insert_doc("Property Setter", { + "doctype_or_field": "DocField", + "doc_type": "ToDo", + "field_name": "assigned_by", + "property": "default", + "property_type": "Text", + "value": "Administrator" + }, true); + cy.reload(); + cy.new_form("ToDo"); + cy.fill_field("description", "new", "Text Editor"); + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); + cy.findByRole("button", {name: "Save"}).click(); + cy.wait("@save_form"); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", "Administrator" + ); + // if user clears default value explicitly, system should not reset default again + cy.get_field("assigned_by").clear().blur(); + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); + cy.findByRole("button", {name: "Save"}).click(); + cy.wait("@save_form"); + cy.get_field("assigned_by").should("have.value", ""); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", "" + ); + }); }); diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 758b3cde2b..4f273af21f 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -110,34 +110,6 @@ Cypress.Commands.add('get_doc', (doctype, name) => { }); }); -Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { - return cy - .window() - .its('frappe.csrf_token') - .then(csrf_token => { - return cy - .request({ - method: 'POST', - url: `/api/resource/${doctype}`, - body: args, - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - }, - failOnStatusCode: !ignore_duplicate - }) - .then(res => { - let status_codes = [200]; - if (ignore_duplicate) { - status_codes.push(409); - } - expect(res.status).to.be.oneOf(status_codes); - return res.body; - }); - }); -}); - Cypress.Commands.add('remove_doc', (doctype, name) => { return cy .window() diff --git a/frappe/__init__.py b/frappe/__init__.py index c6cbfead43..3558603454 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -143,6 +143,8 @@ lang = local("lang") # This if block is never executed when running the code. It is only used for # telling static code analyzer where to find dynamically defined attributes. if typing.TYPE_CHECKING: + from frappe.utils.redis_wrapper import RedisWrapper + from frappe.database.mariadb.database import MariaDBDatabase from frappe.database.postgres.database import PostgresDatabase from frappe.query_builder.builder import MariaDB, Postgres @@ -150,6 +152,7 @@ if typing.TYPE_CHECKING: db: typing.Union[MariaDBDatabase, PostgresDatabase] qb: typing.Union[MariaDB, Postgres] + # end: static analysis hack def init(site, sites_path=None, new_site=False): @@ -311,9 +314,8 @@ def destroy(): release_local(local) -# memcache redis_server = None -def cache(): +def cache() -> "RedisWrapper": """Returns redis connection.""" global redis_server if not redis_server: diff --git a/frappe/api.py b/frappe/api.py index b061761d10..e7f7bf5a04 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -94,7 +94,8 @@ def handle(): "data": doc.save().as_dict() }) - if doc.parenttype and doc.parent: + # check for child table doctype + if doc.get("parenttype"): frappe.get_doc(doc.parenttype, doc.parent).save() frappe.db.commit() diff --git a/frappe/app.py b/frappe/app.py index d73dd67983..609a8535d7 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -192,12 +192,7 @@ def make_form_dict(request): if not isinstance(args, dict): frappe.throw(_("Invalid request arguments")) - try: - frappe.local.form_dict = frappe._dict({ - k: v[0] if isinstance(v, (list, tuple)) else v for k, v in args.items() - }) - except IndexError: - frappe.local.form_dict = frappe._dict(args) + frappe.local.form_dict = frappe._dict(args) if "_" in frappe.local.form_dict: # _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict diff --git a/frappe/client.py b/frappe/client.py index e835e7fee7..1898994afe 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -99,7 +99,6 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren if not filters: filters = None - if frappe.get_meta(doctype).issingle: value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug) else: @@ -129,7 +128,7 @@ def set_value(doctype, name, fieldname, value=None): :param fieldname: fieldname string or JSON / dict with key value pair :param value: value if fieldname is JSON / dict''' - if fieldname!="idx" and fieldname in frappe.model.default_fields: + if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): frappe.throw(_("Cannot edit standard fields")) if not value: @@ -142,14 +141,15 @@ def set_value(doctype, name, fieldname, value=None): else: values = {fieldname: value} - doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) - if doc and doc.parent and doc.parenttype: + # check for child table doctype + if not frappe.get_meta(doctype).istable: + doc = frappe.get_doc(doctype, name) + doc.update(values) + else: + doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) doc = frappe.get_doc(doc.parenttype, doc.parent) child = doc.getone({"doctype": doctype, "name": name}) child.update(values) - else: - doc = frappe.get_doc(doctype, name) - doc.update(values) doc.save() @@ -163,10 +163,10 @@ def insert(doc=None): if isinstance(doc, str): doc = json.loads(doc) - if doc.get("parent") and doc.get("parenttype"): + if doc.get("parenttype"): # inserting a child record - parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent")) - parent.append(doc.get("parentfield"), doc) + parent = frappe.get_doc(doc.parenttype, doc.parent) + parent.append(doc.parentfield, doc) parent.save() return parent.as_dict() else: @@ -187,10 +187,10 @@ def insert_many(docs=None): frappe.throw(_('Only 200 inserts allowed in one request')) for doc in docs: - if doc.get("parent") and doc.get("parenttype"): + if doc.get("parenttype"): # inserting a child record - parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent")) - parent.append(doc.get("parentfield"), doc) + parent = frappe.get_doc(doc.parenttype, doc.parent) + parent.append(doc.parentfield, doc) parent.save() out.append(parent.name) else: diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 41b607b192..e3379a43aa 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -623,6 +623,7 @@ def transform_database(context, table, engine, row_format, failfast): @click.command('run-tests') @click.option('--app', help="For App") @click.option('--doctype', help="For DocType") +@click.option('--case', help="Select particular TestCase") @click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt") @click.option('--test', multiple=True, help="Specific test") @click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests") @@ -636,7 +637,7 @@ def transform_database(context, table, engine, row_format, failfast): @pass_context def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False, coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None, - skip_test_records=False, skip_before_tests=False, failfast=False): + skip_test_records=False, skip_before_tests=False, failfast=False, case=None): with CodeCoverage(coverage, app): import frappe.test_runner @@ -658,7 +659,7 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests, force=context.force, profile=profile, junit_xml_output=junit_xml_output, - ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast) + ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast, case=case) if len(ret.failures) == 0 and len(ret.errors) == 0: ret = 0 diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 216db53c72..dfc560a98a 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -44,6 +44,7 @@ frappe.ui.form.on('Data Import', { } frm.dashboard.show_progress(__('Import Progress'), percent, message); frm.page.set_indicator(__('In Progress'), 'orange'); + frm.trigger('update_primary_action'); // hide progress when complete if (data.current === data.total) { @@ -80,7 +81,10 @@ frappe.ui.form.on('Data Import', { frm.trigger('show_import_log'); frm.trigger('show_import_warnings'); frm.trigger('toggle_submit_after_import'); - frm.trigger('show_import_status'); + + if (frm.doc.status != 'Pending') + frm.trigger('show_import_status'); + frm.trigger('show_report_error_button'); if (frm.doc.status === 'Partial Success') { @@ -128,40 +132,49 @@ frappe.ui.form.on('Data Import', { }, show_import_status(frm) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); - let successful_records = import_log.filter(log => log.success); - let failed_records = import_log.filter(log => !log.success); - if (successful_records.length === 0) return; + frappe.call({ + 'method': 'frappe.core.doctype.data_import.data_import.get_import_status', + 'args': { + 'data_import_name': frm.doc.name + }, + 'callback': function(r) { + let successful_records = cint(r.message.success); + let failed_records = cint(r.message.failed); + let total_records = cint(r.message.total_records); - let message; - if (failed_records.length === 0) { - let message_args = [successful_records.length]; - if (frm.doc.import_type === 'Insert New Records') { - message = - successful_records.length > 1 - ? __('Successfully imported {0} records.', message_args) - : __('Successfully imported {0} record.', message_args); - } else { - message = - successful_records.length > 1 - ? __('Successfully updated {0} records.', message_args) - : __('Successfully updated {0} record.', message_args); + if (!total_records) return; + + let message; + if (failed_records === 0) { + let message_args = [successful_records]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records > 1 + ? __('Successfully imported {0} records.', message_args) + : __('Successfully imported {0} record.', message_args); + } else { + message = + successful_records > 1 + ? __('Successfully updated {0} records.', message_args) + : __('Successfully updated {0} record.', message_args); + } + } else { + let message_args = [successful_records, total_records]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records > 1 + ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) + : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + } else { + message = + successful_records > 1 + ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) + : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + } + } + frm.dashboard.set_headline(message); } - } else { - let message_args = [successful_records.length, import_log.length]; - if (frm.doc.import_type === 'Insert New Records') { - message = - successful_records.length > 1 - ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); - } else { - message = - successful_records.length > 1 - ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); - } - } - frm.dashboard.set_headline(message); + }); }, show_report_error_button(frm) { @@ -275,7 +288,7 @@ frappe.ui.form.on('Data Import', { }, show_import_preview(frm, preview_data) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); + let import_log = preview_data.import_log; if ( frm.import_preview && @@ -316,6 +329,15 @@ frappe.ui.form.on('Data Import', { ); }, + export_import_log(frm) { + open_url_post( + '/api/method/frappe.core.doctype.data_import.data_import.download_import_log', + { + data_import_name: frm.doc.name + } + ); + }, + show_import_warnings(frm, preview_data) { let columns = preview_data.columns; let warnings = JSON.parse(frm.doc.template_warnings || '[]'); @@ -391,92 +413,131 @@ frappe.ui.form.on('Data Import', { frm.trigger('show_import_log'); }, - show_import_log(frm) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); - let logs = import_log; - frm.toggle_display('import_log', false); - frm.toggle_display('import_log_section', logs.length > 0); + render_import_log(frm) { + frappe.call({ + 'method': 'frappe.client.get_list', + 'args': { + 'doctype': 'Data Import Log', + 'filters': { + 'data_import': frm.doc.name + }, + 'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'], + 'limit_page_length': 5000, + 'order_by': 'log_index' + }, + callback: function(r) { + let logs = r.message; - if (logs.length === 0) { - frm.get_field('import_log_preview').$wrapper.empty(); + if (logs.length === 0) return; + + frm.toggle_display('import_log_section', true); + + let rows = logs + .map(log => { + let html = ''; + if (log.success) { + if (frm.doc.import_type === 'Insert New Records') { + html = __('Successfully imported {0}', [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}` + ]); + } else { + html = __('Successfully updated {0}', [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}` + ]); + } + } else { + let messages = (JSON.parse(log.messages || '[]')) + .map(JSON.parse) + .map(m => { + let title = m.title ? `${m.title}` : ''; + let message = m.message ? `
${m.message}
` : ''; + return title + message; + }) + .join(''); + let id = frappe.dom.get_unique_id(); + html = `${messages} + +
+
+ 
${log.exception}
+
+
`; + } + let indicator_color = log.success ? 'green' : 'red'; + let title = log.success ? __('Success') : __('Failure'); + + if (frm.doc.show_failed_logs && log.success) { + return ''; + } + + return ` + ${JSON.parse(log.row_indexes).join(', ')} + +
${title}
+ + + ${html} + + `; + }) + .join(''); + + if (!rows && frm.doc.show_failed_logs) { + rows = ` + ${__('No failed logs')} + `; + } + + frm.get_field('import_log_preview').$wrapper.html(` + + + + + + + ${rows} +
${__('Row Number')}${__('Status')}${__('Message')}
+ `); + } + }); + }, + + show_import_log(frm) { + frm.toggle_display('import_log_section', false); + + if (frm.import_in_progress) { return; } - let rows = logs - .map(log => { - let html = ''; - if (log.success) { - if (frm.doc.import_type === 'Insert New Records') { - html = __('Successfully imported {0}', [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}` - ]); - } else { - html = __('Successfully updated {0}', [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}` - ]); - } + frappe.call({ + 'method': 'frappe.client.get_count', + 'args': { + 'doctype': 'Data Import Log', + 'filters': { + 'data_import': frm.doc.name + } + }, + 'callback': function(r) { + let count = r.message; + if (count < 5000) { + frm.trigger('render_import_log'); } else { - let messages = log.messages - .map(JSON.parse) - .map(m => { - let title = m.title ? `${m.title}` : ''; - let message = m.message ? `
${m.message}
` : ''; - return title + message; - }) - .join(''); - let id = frappe.dom.get_unique_id(); - html = `${messages} - -
-
- 
${log.exception}
-
-
`; + frm.toggle_display('import_log_section', false); + frm.add_custom_button(__('Export Import Log'), () => + frm.trigger('export_import_log') + ); } - let indicator_color = log.success ? 'green' : 'red'; - let title = log.success ? __('Success') : __('Failure'); - - if (frm.doc.show_failed_logs && log.success) { - return ''; - } - - return ` - ${log.row_indexes.join(', ')} - -
${title}
- - - ${html} - - `; - }) - .join(''); - - if (!rows && frm.doc.show_failed_logs) { - rows = ` - ${__('No failed logs')} - `; - } - - frm.get_field('import_log_preview').$wrapper.html(` - - - - - - - ${rows} -
${__('Row Number')}${__('Status')}${__('Message')}
- `); + } + }); }, }); diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index fe6fb90481..9e948dac8c 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -1,194 +1,197 @@ { - "actions": [], - "autoname": "format:{reference_doctype} Import on {creation}", - "beta": 1, - "creation": "2019-08-04 14:16:08.318714", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "import_type", - "download_template", - "import_file", - "html_5", - "google_sheets_url", - "refresh_google_sheet", - "column_break_5", - "status", - "submit_after_import", - "mute_emails", - "template_options", - "import_warnings_section", - "template_warnings", - "import_warnings", - "section_import_preview", - "import_preview", - "import_log_section", - "import_log", - "show_failed_logs", - "import_log_preview" - ], - "fields": [ - { - "fieldname": "reference_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1, - "set_only_once": 1 - }, - { - "fieldname": "import_type", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Import Type", - "options": "\nInsert New Records\nUpdate Existing Records", - "reqd": 1, - "set_only_once": 1 - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "import_file", - "fieldtype": "Attach", - "in_list_view": 1, - "label": "Import File", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "fieldname": "import_preview", - "fieldtype": "HTML", - "label": "Import Preview" - }, - { - "fieldname": "section_import_preview", - "fieldtype": "Section Break", - "label": "Preview" - }, - { - "fieldname": "column_break_5", - "fieldtype": "Column Break" - }, - { - "fieldname": "template_options", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Options", - "options": "JSON", - "read_only": 1 - }, - { - "fieldname": "import_log", - "fieldtype": "Code", - "label": "Import Log", - "options": "JSON" - }, - { - "fieldname": "import_log_section", - "fieldtype": "Section Break", - "label": "Import Log" - }, - { - "fieldname": "import_log_preview", - "fieldtype": "HTML", - "label": "Import Log Preview" - }, - { - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 1, - "label": "Status", - "options": "Pending\nSuccess\nPartial Success\nError", - "read_only": 1 - }, - { - "fieldname": "template_warnings", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Warnings", - "options": "JSON" - }, - { - "default": "0", - "fieldname": "submit_after_import", - "fieldtype": "Check", - "label": "Submit After Import", - "set_only_once": 1 - }, - { - "fieldname": "import_warnings_section", - "fieldtype": "Section Break", - "label": "Import File Errors and Warnings" - }, - { - "fieldname": "import_warnings", - "fieldtype": "HTML", - "label": "Import Warnings" - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "download_template", - "fieldtype": "Button", - "label": "Download Template" - }, - { - "default": "1", - "fieldname": "mute_emails", - "fieldtype": "Check", - "label": "Don't Send Emails", - "set_only_once": 1 - }, - { - "default": "0", - "fieldname": "show_failed_logs", - "fieldtype": "Check", - "label": "Show Failed Logs" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file", - "fieldname": "html_5", - "fieldtype": "HTML", - "options": "
Or
" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file\n", - "description": "Must be a publicly accessible Google Sheets URL", - "fieldname": "google_sheets_url", - "fieldtype": "Data", - "label": "Import from Google Sheets", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", - "fieldname": "refresh_google_sheet", - "fieldtype": "Button", - "label": "Refresh Google Sheet" - } - ], - "hide_toolbar": 1, - "links": [], - "modified": "2021-04-11 01:50:42.074623", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 + "actions": [], + "autoname": "format:{reference_doctype} Import on {creation}", + "beta": 1, + "creation": "2019-08-04 14:16:08.318714", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "import_type", + "download_template", + "import_file", + "payload_count", + "html_5", + "google_sheets_url", + "refresh_google_sheet", + "column_break_5", + "status", + "submit_after_import", + "mute_emails", + "template_options", + "import_warnings_section", + "template_warnings", + "import_warnings", + "section_import_preview", + "import_preview", + "import_log_section", + "show_failed_logs", + "import_log_preview" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "import_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Import Type", + "options": "\nInsert New Records\nUpdate Existing Records", + "reqd": 1, + "set_only_once": 1 + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "import_file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Import File", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "fieldname": "import_preview", + "fieldtype": "HTML", + "label": "Import Preview" + }, + { + "fieldname": "section_import_preview", + "fieldtype": "Section Break", + "label": "Preview" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "template_options", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Options", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "import_log_section", + "fieldtype": "Section Break", + "label": "Import Log" + }, + { + "fieldname": "import_log_preview", + "fieldtype": "HTML", + "label": "Import Log Preview" + }, + { + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "label": "Status", + "options": "Pending\nSuccess\nPartial Success\nError", + "read_only": 1 + }, + { + "fieldname": "template_warnings", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Warnings", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "label": "Submit After Import", + "set_only_once": 1 + }, + { + "fieldname": "import_warnings_section", + "fieldtype": "Section Break", + "label": "Import File Errors and Warnings" + }, + { + "fieldname": "import_warnings", + "fieldtype": "HTML", + "label": "Import Warnings" + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" + }, + { + "default": "1", + "fieldname": "mute_emails", + "fieldtype": "Check", + "label": "Don't Send Emails", + "set_only_once": 1 + }, + { + "default": "0", + "fieldname": "show_failed_logs", + "fieldtype": "Check", + "label": "Show Failed Logs" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file", + "fieldname": "html_5", + "fieldtype": "HTML", + "options": "
Or
" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file\n", + "description": "Must be a publicly accessible Google Sheets URL", + "fieldname": "google_sheets_url", + "fieldtype": "Data", + "label": "Import from Google Sheets", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", + "fieldname": "refresh_google_sheet", + "fieldtype": "Button", + "label": "Refresh Google Sheet" + }, + { + "fieldname": "payload_count", + "fieldtype": "Int", + "hidden": 1, + "label": "Payload Count", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "links": [], + "modified": "2022-02-01 20:08:37.624914", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 5935ddc4ba..5972e79b4d 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -27,6 +27,7 @@ class DataImport(Document): self.validate_import_file() self.validate_google_sheets_url() + self.set_payload_count() def validate_import_file(self): if self.import_file: @@ -38,6 +39,12 @@ class DataImport(Document): return validate_google_sheets_url(self.google_sheets_url) + def set_payload_count(self): + if self.import_file: + i = self.get_importer() + payloads = i.import_file.get_payloads_for_import() + self.payload_count = len(payloads) + @frappe.whitelist() def get_preview_from_template(self, import_file=None, google_sheets_url=None): if import_file: @@ -67,7 +74,7 @@ class DataImport(Document): enqueue( start_import, queue="default", - timeout=6000, + timeout=10000, event="data_import", job_name=self.name, data_import=self.name, @@ -80,6 +87,9 @@ class DataImport(Document): def export_errored_rows(self): return self.get_importer().export_errored_rows() + def download_import_log(self): + return self.get_importer().export_import_log() + def get_importer(self): return Importer(self.reference_doctype, data_import=self) @@ -90,7 +100,6 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N import_file, google_sheets_url ) - @frappe.whitelist() def form_start_import(data_import): return frappe.get_doc("Data Import", data_import).start_import() @@ -145,6 +154,30 @@ def download_errored_template(data_import_name): data_import = frappe.get_doc("Data Import", data_import_name) data_import.export_errored_rows() +@frappe.whitelist() +def download_import_log(data_import_name): + data_import = frappe.get_doc("Data Import", data_import_name) + data_import.download_import_log() + +@frappe.whitelist() +def get_import_status(data_import_name): + import_status = {} + + logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'], + filters={'data_import': data_import_name}, + group_by='success') + + total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count') + + for log in logs: + if log.get('success'): + import_status['success'] = log.get('count') + else: + import_status['failed'] = log.get('count') + + import_status['total_records'] = total_payload_count + + return import_status def import_file( doctype, file_path, import_type, submit_after_import=False, console=False diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 0eb05aa354..6ab750ba25 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -24,12 +24,14 @@ frappe.listview_settings['Data Import'] = { 'Error': 'red' }; let status = doc.status; + if (imports_in_progress.includes(doc.name)) { status = 'In Progress'; } if (status == 'Pending') { status = 'Not Started'; } + return [__(status), colors[status], 'status,=,' + doc.status]; }, formatters: { diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index b9b2050763..f085709945 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -47,7 +47,13 @@ class Importer: ) def get_data_for_import_preview(self): - return self.import_file.get_data_for_import_preview() + out = self.import_file.get_data_for_import_preview() + + out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index", limit=10) + + return out def before_import(self): # set user lang for translations @@ -58,7 +64,6 @@ class Importer: frappe.flags.in_import = True frappe.flags.mute_emails = self.data_import.mute_emails - self.data_import.db_set("status", "Pending") self.data_import.db_set("template_warnings", "") def import_data(self): @@ -79,20 +84,25 @@ class Importer: return # setup import log - if self.data_import.import_log: - import_log = frappe.parse_json(self.data_import.import_log) - else: - import_log = [] + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] - # remove previous failures from import log - import_log = [log for log in import_log if log.get("success")] + log_index = 0 + + # Do not remove rows in case of retry after an error or pending data import + if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count: + # remove previous failures from import log only in case of retry after partial success + import_log = [log for log in import_log if log.get("success")] # get successfully imported rows imported_rows = [] for log in import_log: log = frappe._dict(log) - if log.success: - imported_rows += log.row_indexes + if log.success or len(import_log) < self.data_import.payload_count: + imported_rows += json.loads(log.row_indexes) + + log_index = log.log_index # start import total_payload_count = len(payloads) @@ -146,25 +156,41 @@ class Importer: }, ) - import_log.append( - frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) - ) + create_import_log(self.data_import.name, log_index, { + 'success': True, + 'docname': doc.name, + 'row_indexes': row_indexes + }) + + log_index += 1 + + if not self.data_import.status == "Partial Success": + self.data_import.db_set("status", "Partial Success") + # commit after every successful import frappe.db.commit() except Exception: - import_log.append( - frappe._dict( - success=False, - exception=frappe.get_traceback(), - messages=frappe.local.message_log, - row_indexes=row_indexes, - ) - ) + messages = frappe.local.message_log frappe.clear_messages() + # rollback if exception frappe.db.rollback() + create_import_log(self.data_import.name, log_index, { + 'success': False, + 'exception': frappe.get_traceback(), + 'messages': messages, + 'row_indexes': row_indexes + }) + + log_index += 1 + + # Logs are db inserted directly so will have to be fetched again + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + # set status failures = [log for log in import_log if not log.get("success")] if len(failures) == total_payload_count: @@ -178,7 +204,6 @@ class Importer: self.print_import_log(import_log) else: self.data_import.db_set("status", status) - self.data_import.db_set("import_log", json.dumps(import_log)) self.after_import() @@ -248,11 +273,14 @@ class Importer: if not self.data_import: return - import_log = frappe.parse_json(self.data_import.import_log or "[]") + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + failures = [log for log in import_log if not log.get("success")] row_indexes = [] for f in failures: - row_indexes.extend(f.get("row_indexes", [])) + row_indexes.extend(json.loads(f.get("row_indexes", []))) # de duplicate row_indexes = list(set(row_indexes)) @@ -264,6 +292,30 @@ class Importer: build_csv_response(rows, _(self.doctype)) + def export_import_log(self): + from frappe.utils.csvutils import build_csv_response + + if not self.data_import: + return + + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": self.data_import.name}, + order_by="log_index") + + header_row = ["Row Numbers", "Status", "Message", "Exception"] + + rows = [header_row] + + for log in import_log: + row_number = json.loads(log.get("row_indexes"))[0] + status = "Success" if log.get('success') else "Failure" + message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \ + log.get("messages") + exception = frappe.utils.cstr(log.get("exception", '')) + rows += [[row_number, status, message, exception]] + + build_csv_response(rows, self.doctype) + def print_import_log(self, import_log): failed_records = [log for log in import_log if not log.success] successful_records = [log for log in import_log if log.success] @@ -566,7 +618,7 @@ class Row: ) # remove standard fields and __islocal - for key in frappe.model.default_fields + ("__islocal",): + for key in frappe.model.default_fields + frappe.model.child_table_fields + ("__islocal",): doc.pop(key, None) for col, value in zip(columns, values): @@ -1172,3 +1224,17 @@ def df_as_json(df): def get_select_options(df): return [d for d in (df.options or "").split("\n") if d] + +def create_import_log(data_import, log_index, log_details): + frappe.get_doc({ + 'doctype': 'Data Import Log', + 'log_index': log_index, + 'success': log_details.get('success'), + 'data_import': data_import, + 'row_indexes': json.dumps(log_details.get('row_indexes')), + 'docname': log_details.get('docname'), + 'messages': json.dumps(log_details.get('messages', '[]')), + 'exception': log_details.get('exception') + }).db_insert() + + diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index e1bc0e7ca5..11077ca58b 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -4,6 +4,7 @@ import unittest import frappe from frappe.core.doctype.data_import.importer import Importer +from frappe.tests.test_query_builder import db_type_is, run_only_if from frappe.utils import getdate, format_duration doctype_name = 'DocType for Import' @@ -54,21 +55,27 @@ class TestImporter(unittest.TestCase): self.assertEqual(len(preview.data), 4) self.assertEqual(len(preview.columns), 16) + # ignored on postgres because myisam doesn't exist on pg + @run_only_if(db_type_is.MARIADB) def test_data_import_without_mandatory_values(self): import_file = get_import_file('sample_import_file_without_mandatory') data_import = self.get_importer(doctype_name, import_file) frappe.local.message_log = [] data_import.start_import() data_import.reload() - import_log = frappe.parse_json(data_import.import_log) - self.assertEqual(import_log[0]['row_indexes'], [2,3]) - expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error) - expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error) - self.assertEqual(import_log[1]['row_indexes'], [4]) - self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required") + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": data_import.name}, + order_by="log_index") + + self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3]) + expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error) + expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error) + + self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4]) + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required") def test_data_import_update(self): existing_doc = frappe.get_doc( diff --git a/frappe/core/doctype/data_import_log/__init__.py b/frappe/core/doctype/data_import_log/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/data_import_log/data_import_log.js b/frappe/core/doctype/data_import_log/data_import_log.js new file mode 100644 index 0000000000..c376edeec9 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2021, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Data Import Log', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/data_import_log/data_import_log.json b/frappe/core/doctype/data_import_log/data_import_log.json new file mode 100644 index 0000000000..b1d991f099 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.json @@ -0,0 +1,84 @@ +{ + "actions": [], + "creation": "2021-12-25 16:12:20.205889", + "doctype": "DocType", + "editable_grid": 1, + "engine": "MyISAM", + "field_order": [ + "data_import", + "row_indexes", + "success", + "docname", + "messages", + "exception", + "log_index" + ], + "fields": [ + { + "fieldname": "data_import", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Data Import", + "options": "Data Import" + }, + { + "fieldname": "docname", + "fieldtype": "Data", + "label": "Reference Name" + }, + { + "fieldname": "exception", + "fieldtype": "Text", + "label": "Exception" + }, + { + "fieldname": "row_indexes", + "fieldtype": "Code", + "label": "Row Indexes", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "success", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Success" + }, + { + "fieldname": "log_index", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Log Index" + }, + { + "fieldname": "messages", + "fieldtype": "Code", + "label": "Messages", + "options": "JSON" + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2021-12-29 11:19:19.646076", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import Log", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC" +} \ No newline at end of file diff --git a/frappe/core/doctype/data_import_log/data_import_log.py b/frappe/core/doctype/data_import_log/data_import_log.py new file mode 100644 index 0000000000..a71aefa8bc --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class DataImportLog(Document): + pass diff --git a/frappe/core/doctype/data_import_log/test_data_import_log.py b/frappe/core/doctype/data_import_log/test_data_import_log.py new file mode 100644 index 0000000000..244404936e --- /dev/null +++ b/frappe/core/doctype/data_import_log/test_data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and Contributors +# See license.txt + +# import frappe +import unittest + +class TestDataImportLog(unittest.TestCase): + pass diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3754288145..d259367a16 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -10,7 +10,9 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache import frappe from frappe import _ from frappe.utils import now, cint -from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options +from frappe.model import ( + no_value_fields, default_fields, table_fields, data_field_options, child_table_fields +) from frappe.model.document import Document from frappe.model.base_document import get_controller from frappe.custom.doctype.property_setter.property_setter import make_property_setter @@ -74,6 +76,7 @@ class DocType(Document): self.make_amendable() self.make_repeatable() self.validate_nestedset() + self.validate_child_table() self.validate_website() self.ensure_minimum_max_attachment_limit() validate_links_table_fieldnames(self) @@ -689,6 +692,22 @@ class DocType(Document): }) self.nsm_parent_field = parent_field_name + def validate_child_table(self): + if not self.get("istable") or self.is_new(): + # if the doctype is not a child table then return + # if the doctype is a new doctype and also a child table then + # don't move forward as it will be handled via schema + return + + self.add_child_table_fields() + + def add_child_table_fields(self): + from frappe.database.schema import add_column + + add_column(self.name, "parent", "Data") + add_column(self.name, "parenttype", "Data") + add_column(self.name, "parentfield", "Data") + def get_max_idx(self): """Returns the highest `idx`""" max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", @@ -699,6 +718,13 @@ class DocType(Document): if not name: name = self.name + # a Doctype name is the tablename created in database + # `tab` the length of tablename is limited to 64 characters + max_length = frappe.db.MAX_COLUMN_LENGTH - 3 + if len(name) > max_length: + # length(tab + ) should be equal to 64 characters hence doctype should be 61 characters + frappe.throw(_("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError) + flags = {"flags": re.ASCII} # a DocType name should not start or end with an empty space @@ -1009,7 +1035,7 @@ def validate_fields(meta): sort_fields = [d.split()[0] for d in meta.sort_field.split(',')] for fieldname in sort_fields: - if not fieldname in fieldname_list + list(default_fields): + if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)): frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname), InvalidFieldNameError) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 12c227464d..50882f51bd 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -23,6 +23,7 @@ class TestDocType(unittest.TestCase): self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert) self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert) self.assertRaises(frappe.NameError, new_doctype("Some (DocType)").insert) + self.assertRaises(frappe.NameError, new_doctype("Some Doctype with a name whose length is more than 61 characters").insert) for name in ("Some DocType", "Some_DocType"): if frappe.db.exists("DocType", name): frappe.delete_doc("DocType", name) @@ -353,7 +354,6 @@ class TestDocType(unittest.TestCase): dump_docs = json.dumps(docs.get('docs')) cancel_all_linked_docs(dump_docs) data_link_doc.cancel() - data_doc.name = '{}-CANC-0'.format(data_doc.name) data_doc.load_from_db() self.assertEqual(data_link_doc.docstatus, 2) self.assertEqual(data_doc.docstatus, 2) @@ -377,7 +377,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - link_doc.insert(ignore_if_duplicate=True) + link_doc.insert() #create first parent doctype test_doc_1 = new_doctype('Test Doctype 1') @@ -392,7 +392,7 @@ class TestDocType(unittest.TestCase): for data in test_doc_1.get('permissions'): data.submit = 1 data.cancel = 1 - test_doc_1.insert(ignore_if_duplicate=True) + test_doc_1.insert() #crete second parent doctype doc = new_doctype('Test Doctype 2') @@ -407,7 +407,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - doc.insert(ignore_if_duplicate=True) + doc.insert() # create doctype data data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') @@ -438,7 +438,6 @@ class TestDocType(unittest.TestCase): # checking that doc for Test Doctype 2 is not canceled self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) - data_doc_2.name = '{}-CANC-0'.format(data_doc_2.name) data_doc.load_from_db() data_doc_2.load_from_db() self.assertEqual(data_link_doc_1.docstatus, 2) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index d9c966cc95..2808a2710b 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE """ @@ -7,7 +7,6 @@ record of files naming for same name files: file.gif, file-1.gif, file-2.gif etc """ -import base64 import hashlib import imghdr import io @@ -17,9 +16,10 @@ import os import re import shutil import zipfile +from typing import TYPE_CHECKING, Tuple import requests -import requests.exceptions +from requests.exceptions import HTTPError, SSLError from PIL import Image, ImageFile, ImageOps from io import BytesIO from urllib.parse import quote, unquote @@ -31,6 +31,11 @@ from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, g from frappe.utils.image import strip_exif_data, optimize_image from frappe.utils.file_manager import safe_b64decode +if TYPE_CHECKING: + from PIL.ImageFile import ImageFile + from requests.models import Response + + class MaxFileSizeReachedError(frappe.ValidationError): pass @@ -276,7 +281,7 @@ class File(Document): image, filename, extn = get_local_image(self.file_url) else: image, filename, extn = get_web_image(self.file_url) - except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError): + except (HTTPError, SSLError, IOError, TypeError): return size = width, height @@ -648,9 +653,17 @@ def setup_folder_path(filename, new_parent): from frappe.model.rename_doc import rename_doc rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True) -def get_extension(filename, extn, content): +def get_extension(filename, extn, content: bytes = None, response: "Response" = None) -> str: mimetype = None + if response: + content_type = response.headers.get("Content-Type") + + if content_type: + _extn = mimetypes.guess_extension(content_type) + if _extn: + return _extn[1:] + if extn: # remove '?' char and parameters from extn if present if '?' in extn: @@ -693,14 +706,14 @@ def get_local_image(file_url): return image, filename, extn -def get_web_image(file_url): +def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]: # download file_url = frappe.utils.get_url(file_url) r = requests.get(file_url, stream=True) try: r.raise_for_status() - except requests.exceptions.HTTPError as e: - if "404" in e.args[0]: + except HTTPError: + if r.status_code == 404: frappe.msgprint(_("File '{0}' not found").format(file_url)) else: frappe.msgprint(_("Unable to read file format for {0}").format(file_url)) @@ -719,7 +732,10 @@ def get_web_image(file_url): filename = get_random_filename() extn = None - extn = get_extension(filename, extn, r.content) + extn = get_extension(filename, extn, response=r) + if extn == "bin": + extn = get_extension(filename, extn, content=r.content) or "png" + filename = "/files/" + strip(unquote(filename)) return image, filename, extn @@ -862,8 +878,9 @@ def extract_images_from_html(doc, content, is_private=False): else: filename = get_random_filename(content_type=mtype) - doctype = doc.parenttype if doc.parent else doc.doctype - name = doc.parent or doc.name + # attaching a file to a child table doc, attaches it to the parent doc + doctype = doc.parenttype if doc.get("parent") else doc.doctype + name = doc.get("parent") or doc.name _file = frappe.get_doc({ "doctype": "File", diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2c1042e104..ba83dfca19 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import base64 import json import frappe import os import unittest + from frappe import _ -from frappe.core.doctype.file.file import get_attached_images, move_file, get_files_in_folder, unzip_file +from frappe.core.doctype.file.file import File, get_attached_images, move_file, get_files_in_folder, unzip_file from frappe.utils import get_files_path -# test_records = frappe.get_test_records('File') test_content1 = 'Hello' test_content2 = 'Hello World' @@ -24,8 +23,6 @@ def make_test_doc(): class TestSimpleFile(unittest.TestCase): - - def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = test_content1 @@ -38,21 +35,13 @@ class TestSimpleFile(unittest.TestCase): _file.save() self.saved_file_url = _file.file_url - def test_save(self): _file = frappe.get_doc("File", {"file_url": self.saved_file_url}) content = _file.get_content() self.assertEqual(content, self.test_content) - def tearDown(self): - # File gets deleted on rollback, so blank - pass - - class TestBase64File(unittest.TestCase): - - def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = base64.b64encode(test_content1.encode('utf-8')) @@ -66,18 +55,12 @@ class TestBase64File(unittest.TestCase): _file.save() self.saved_file_url = _file.file_url - def test_saved_content(self): _file = frappe.get_doc("File", {"file_url": self.saved_file_url}) content = _file.get_content() self.assertEqual(content, test_content1) - def tearDown(self): - # File gets deleted on rollback, so blank - pass - - class TestSameFileName(unittest.TestCase): def test_saved_content(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() @@ -130,8 +113,6 @@ class TestSameFileName(unittest.TestCase): class TestSameContent(unittest.TestCase): - - def setUp(self): self.attached_to_doctype1, self.attached_to_docname1 = make_test_doc() self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc() @@ -186,10 +167,6 @@ class TestSameContent(unittest.TestCase): limit_property.delete() frappe.clear_cache(doctype='ToDo') - def tearDown(self): - # File gets deleted on rollback, so blank - pass - class TestFile(unittest.TestCase): def setUp(self): @@ -398,7 +375,7 @@ class TestFile(unittest.TestCase): def test_make_thumbnail(self): # test web image - test_file = frappe.get_doc({ + test_file: File = frappe.get_doc({ "doctype": "File", "file_name": 'logo', "file_url": frappe.utils.get_url('/_test/assets/image.jpg'), @@ -407,6 +384,16 @@ class TestFile(unittest.TestCase): test_file.make_thumbnail() self.assertEquals(test_file.thumbnail_url, '/files/image_small.jpg') + # test web image without extension + test_file = frappe.get_doc({ + "doctype": "File", + "file_name": 'logo', + "file_url": frappe.utils.get_url('/_test/assets/image'), + }).insert(ignore_permissions=True) + + test_file.make_thumbnail() + self.assertTrue(test_file.thumbnail_url.endswith("_small.jpeg")) + # test local image test_file.db_set('thumbnail_url', None) test_file.reload() diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 266017dd71..9cb40dffd4 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -61,7 +61,7 @@ class Report(Document): delete_permanently=True) def get_columns(self): - return [d.as_dict(no_default_fields = True) for d in self.columns] + return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns] @frappe.whitelist() def set_doctype_roles(self): diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index bc92061f42..d9381bcd16 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -139,3 +139,42 @@ class TestServerScript(unittest.TestCase): server_script.disabled = 1 server_script.save() + + def test_restricted_qb(self): + todo = frappe.get_doc(doctype="ToDo", description="QbScriptTestNote") + todo.insert() + + script = frappe.get_doc( + doctype='Server Script', + name='test_qb_restrictions', + script_type = 'API', + api_method = 'test_qb_restrictions', + allow_guest = 1, + # whitelisted update + script = f''' +frappe.db.set_value("ToDo", "{todo.name}", "description", "safe") +''' + ) + script.insert() + script.execute_method() + + todo.reload() + self.assertEqual(todo.description, "safe") + + # unsafe update + script.script = f""" +todo = frappe.qb.DocType("ToDo") +frappe.qb.update(todo).set(todo.description, "unsafe").where(todo.name == "{todo.name}").run() +""" + script.save() + self.assertRaises(frappe.PermissionError, script.execute_method) + todo.reload() + self.assertEqual(todo.description, "safe") + + # safe select + script.script = f""" +todo = frappe.qb.DocType("ToDo") +frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run() +""" + script.save() + script.execute_method() diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index cf905c2ce2..d4a9d68fd5 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -3,6 +3,7 @@ from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable from frappe.permissions import has_user_permission from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.website.doctype.blog_post.test_blog_post import make_test_blog import frappe import unittest @@ -31,6 +32,18 @@ class TestUserPermission(unittest.TestCase): param = get_params(user, 'User', perm_user.name, is_default=1) self.assertRaises(frappe.ValidationError, add_user_permissions, param) + def test_default_user_permission_corectness(self): + user = create_user('test_default_corectness_permission_1@example.com') + param = get_params(user, 'User', user.name, is_default=1, hide_descendants= 1) + add_user_permissions(param) + #create a duplicate entry with default + perm_user = create_user('test_default_corectness2@example.com') + test_blog = make_test_blog() + param = get_params(perm_user, 'Blog Post', test_blog.name, is_default=1, hide_descendants= 1) + add_user_permissions(param) + frappe.db.delete('User Permission', filters={'for_value': test_blog.name}) + frappe.delete_doc('Blog Post', test_blog.name) + def test_default_user_permission(self): frappe.set_user('Administrator') user = create_user('test_user_perm1@example.com', 'Website Manager') diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 1366ace115..fb658481b2 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -48,7 +48,6 @@ class UserPermission(Document): }, or_filters={ 'applicable_for': cstr(self.applicable_for), 'apply_to_all_doctypes': 1, - 'hide_descendants': cstr(self.hide_descendants) }, limit=1) if overlap_exists: ref_link = frappe.get_desk_link(self.doctype, overlap_exists[0].name) diff --git a/frappe/core/doctype/user_type/test_user_type.py b/frappe/core/doctype/user_type/test_user_type.py index 7080e1830b..6807f8fc9e 100644 --- a/frappe/core/doctype/user_type/test_user_type.py +++ b/frappe/core/doctype/user_type/test_user_type.py @@ -1,8 +1,68 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE -# import frappe +import frappe import unittest +from frappe.installer import update_site_config + class TestUserType(unittest.TestCase): - pass + def setUp(self): + create_role() + + def test_add_select_perm_doctypes(self): + user_type = create_user_type('Test User Type') + + # select perms added for all link fields + doc = frappe.get_meta('Contact') + link_fields = doc.get_link_fields() + select_doctypes = frappe.get_all('User Select Document Type', {'parent': user_type.name}, pluck='document_type') + + for entry in link_fields: + self.assertTrue(entry.options in select_doctypes) + + # select perms added for all child table link fields + link_fields = [] + for child_table in doc.get_table_fields(): + child_doc = frappe.get_meta(child_table.options) + link_fields.extend(child_doc.get_link_fields()) + + for entry in link_fields: + self.assertTrue(entry.options in select_doctypes) + + def tearDown(self): + frappe.db.rollback() + + +def create_user_type(user_type): + if frappe.db.exists('User Type', user_type): + frappe.delete_doc('User Type', user_type) + + user_type_limit = {frappe.scrub(user_type): 1} + update_site_config('user_type_doctype_limit', user_type_limit) + + doc = frappe.get_doc({ + 'doctype': 'User Type', + 'name': user_type, + 'role': '_Test User Type', + 'user_id_field': 'user', + 'apply_user_permission_on': 'User' + }) + + doc.append('user_doctypes', { + 'document_type': 'Contact', + 'read': 1, + 'write': 1 + }) + + return doc.insert() + + +def create_role(): + if not frappe.db.exists('Role', '_Test User Type'): + frappe.get_doc({ + 'doctype': 'Role', + 'role_name': '_Test User Type', + 'desk_access': 1, + 'is_custom': 1 + }).insert() \ No newline at end of file diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 626ab772b8..c0dfd2e597 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters ['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]] doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters, - order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1) + order_by='`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1) custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)], ['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']] diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index be3e723af6..5f41f217f0 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -39,43 +39,3 @@ def get_todays_events(as_list=False): today = nowdate() events = get_events(today, today) return events if as_list else len(events) - -def get_unseen_likes(): - """Returns count of unseen likes""" - - comment_doctype = DocType("Comment") - return frappe.db.count(comment_doctype, - filters=( - (comment_doctype.comment_type == "Like") - & (comment_doctype.modified >= Now() - Interval(years=1)) - & (comment_doctype.owner.notnull()) - & (comment_doctype.owner != frappe.session.user) - & (comment_doctype.reference_owner == frappe.session.user) - & (comment_doctype.seen == 0) - ) - ) - - -def get_unread_emails(): - "returns count of unread emails for a user" - - communication_doctype = DocType("Communication") - user_doctype = DocType("User") - distinct_email_accounts = ( - frappe.qb.from_(user_doctype) - .select(user_doctype.email_account) - .where(user_doctype.parent == frappe.session.user) - .distinct() - ) - - return frappe.db.count(communication_doctype, - filters=( - (communication_doctype.communication_type == "Communication") - & (communication_doctype.communication_medium == "Email") - & (communication_doctype.sent_or_received == "Received") - & (communication_doctype.email_status.notin(["spam", "Trash"])) - & (communication_doctype.email_account.isin(distinct_email_accounts)) - & (communication_doctype.modified >= Now() - Interval(years=1)) - & (communication_doctype.seen == 0) - ) - ) diff --git a/frappe/core/page/dashboard_view/dashboard_view.js b/frappe/core/page/dashboard_view/dashboard_view.js index e8e9cc9502..bf9fb2a286 100644 --- a/frappe/core/page/dashboard_view/dashboard_view.js +++ b/frappe/core/page/dashboard_view/dashboard_view.js @@ -30,6 +30,7 @@ class Dashboard { show() { this.route = frappe.get_route(); + this.set_breadcrumbs(); if (this.route.length > 1) { // from route this.show_dashboard(this.route.slice(-1)[0]); @@ -75,6 +76,10 @@ class Dashboard { frappe.last_dashboard = current_dashboard_name; } + set_breadcrumbs() { + frappe.breadcrumbs.add("Desk", "Dashboard"); + } + refresh() { frappe.run_serially([ () => this.render_cards(), diff --git a/frappe/database/database.py b/frappe/database/database.py index 65242e0419..9fa1ff161c 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -10,19 +10,20 @@ import re import string from contextlib import contextmanager from time import time -from typing import Dict, List, Union, Tuple +from typing import Dict, List, Tuple, Union + +from pypika.terms import Criterion, NullValue, PseudoColumn import frappe import frappe.defaults import frappe.model.meta from frappe import _ -from frappe.utils import now, getdate, cast, get_datetime from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count -from frappe.query_builder.functions import Min, Max, Avg, Sum -from frappe.query_builder.utils import Column +from frappe.query_builder.utils import DocType +from frappe.utils import cast, get_datetime, getdate, now, sbool + from .query import Query -from pypika.terms import Criterion, PseudoColumn class Database(object): @@ -36,9 +37,9 @@ class Database(object): OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"] DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"] - STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype') - DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent', - 'parentfield', 'parenttype', 'idx'] + STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by') + DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'idx'] + CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield') MAX_WRITES_PER_TRANSACTION = 200_000 class InvalidColumnName(frappe.ValidationError): pass @@ -278,7 +279,9 @@ class Database(object): if self.auto_commit_on_many_writes: self.commit() else: - frappe.throw(_("Too many writes in one request. Please send smaller requests"), frappe.ValidationError) + msg = "

" + _("Too many changes to database in single action.") + "
" + msg += _("The changes have been reverted.") + "
" + raise frappe.TooManyWritesError(msg) def check_implicit_commit(self, query): if self.transaction_writes and \ @@ -432,11 +435,9 @@ class Database(object): else: fields = fieldname - if fieldname!="*": + if fieldname != "*": if isinstance(fieldname, str): fields = [fieldname] - else: - fields = fieldname if (filters is not None) and (filters!=doctype or doctype=="DocType"): try: @@ -555,7 +556,21 @@ class Database(object): def get_list(*args, **kwargs): return frappe.get_list(*args, **kwargs) - def get_single_value(self, doctype, fieldname, cache=False): + def set_single_value(self, doctype, fieldname, value, *args, **kwargs): + """Set field value of Single DocType. + + :param doctype: DocType of the single object + :param fieldname: `fieldname` of the property + :param value: `value` of the property + + Example: + + # Update the `deny_multiple_sessions` field in System Settings DocType. + company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True) + """ + return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs) + + def get_single_value(self, doctype, fieldname, cache=True): """Get property of Single DocType. Cache locally by default :param doctype: DocType of the single object whose value is requested @@ -570,7 +585,7 @@ class Database(object): if not doctype in self.value_cache: self.value_cache[doctype] = {} - if fieldname in self.value_cache[doctype]: + if cache and fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] val = self.query.get_sql( @@ -677,53 +692,55 @@ class Database(object): :param debug: Print the query in the developer / js console. :param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit. """ - if not modified: - modified = now() - if not modified_by: - modified_by = frappe.session.user + is_single_doctype = not (dn and dt != dn) + to_update = field if isinstance(field, dict) else {field: val} - to_update = {} if update_modified: - to_update = {"modified": modified, "modified_by": modified_by} + modified = modified or now() + modified_by = modified_by or frappe.session.user + to_update.update({"modified": modified, "modified_by": modified_by}) + + if is_single_doctype: + frappe.db.delete( + "Singles", + filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug + ) + + singles_data = ((dt, key, sbool(value)) for key, value in to_update.items()) + query = ( + frappe.qb.into("Singles") + .columns("doctype", "field", "value") + .insert(*singles_data) + ).run(debug=debug) + frappe.clear_document_cache(dt, dt) - if isinstance(field, dict): - to_update.update(field) else: - to_update.update({field: val}) + table = DocType(dt) - if dn and dt!=dn: - # with table - set_values = [] - for key in to_update: - set_values.append('`{0}`=%({0})s'.format(key)) + if for_update: + docnames = tuple( + self.get_values(dt, dn, "name", debug=debug, for_update=for_update, pluck=True) + ) or (NullValue(),) + query = frappe.qb.update(table).where(table.name.isin(docnames)) - for name in self.get_values(dt, dn, 'name', for_update=for_update, debug=debug): - values = dict(name=name[0]) - values.update(to_update) + for docname in docnames: + frappe.clear_document_cache(dt, docname) - self.sql("""update `tab{0}` - set {1} where name=%(name)s""".format(dt, ', '.join(set_values)), - values, debug=debug) + else: + query = self.query.build_conditions(table=dt, filters=dn, update=True) + # TODO: Fix this; doesn't work rn - gavin@frappe.io + # frappe.cache().hdel_keys(dt, "document_cache") + # Workaround: clear all document caches + frappe.cache().delete_value('document_cache') - frappe.clear_document_cache(dt, values['name']) - else: - # for singles - keys = list(to_update) - self.sql(''' - delete from `tabSingles` - where field in ({0}) and - doctype=%s'''.format(', '.join(['%s']*len(keys))), - list(keys) + [dt], debug=debug) - for key, value in to_update.items(): - self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''', - (dt, key, value), debug=debug) + for column, value in to_update.items(): + query = query.set(column, value) - frappe.clear_document_cache(dt, dn) + query.run(debug=debug) if dt in self.value_cache: del self.value_cache[dt] - @staticmethod def set(doc, field, val): """Set value in document. **Avoid**""" diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index cfb4e243a2..7c9309ee9f 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -171,9 +171,6 @@ CREATE TABLE `tabDocType` ( `modified_by` varchar(255) DEFAULT NULL, `owner` varchar(255) DEFAULT NULL, `docstatus` int(1) NOT NULL DEFAULT 0, - `parent` varchar(255) DEFAULT NULL, - `parentfield` varchar(255) DEFAULT NULL, - `parenttype` varchar(255) DEFAULT NULL, `idx` int(8) NOT NULL DEFAULT 0, `search_fields` varchar(255) DEFAULT NULL, `issingle` int(1) NOT NULL DEFAULT 0, @@ -228,8 +225,7 @@ CREATE TABLE `tabDocType` ( `subject_field` varchar(255) DEFAULT NULL, `sender_field` varchar(255) DEFAULT NULL, `migration_hash` varchar(255) DEFAULT NULL, - PRIMARY KEY (`name`), - KEY `parent` (`parent`) + PRIMARY KEY (`name`) ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 07bb4d5d7c..fd4bfc6dd0 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -18,6 +18,17 @@ class MariaDBTable(DBTable): if index_defs: additional_definitions += ',\n'.join(index_defs) + ',\n' + # child table columns + if self.meta.get("istable") or 0: + additional_definitions += ',\n'.join( + ( + f"parent varchar({varchar_len})", + f"parentfield varchar({varchar_len})", + f"parenttype varchar({varchar_len})", + "index parent(parent)" + ) + ) + ',\n' + # create table query = f"""create table `{self.table_name}` ( name varchar({varchar_len}) not null primary key, @@ -26,12 +37,8 @@ class MariaDBTable(DBTable): modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus int(1) not null default '0', - parent varchar({varchar_len}), - parentfield varchar({varchar_len}), - parenttype varchar({varchar_len}), idx int(8) not null default '0', {additional_definitions} - index parent(parent), index modified(modified)) ENGINE={engine} ROW_FORMAT=DYNAMIC diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index d5495c6879..a3266242a5 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -170,11 +170,11 @@ class PostgresDatabase(Database): @staticmethod def is_primary_key_violation(e): - return e.pgcode == '23505' and '_pkey' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == '23505' and '_pkey' in cstr(e.args[0]) @staticmethod def is_unique_key_violation(e): - return e.pgcode == '23505' and '_key' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == '23505' and '_key' in cstr(e.args[0]) @staticmethod def is_duplicate_fieldname(e): diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index f911e34650..1662b7b93e 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -176,9 +176,6 @@ CREATE TABLE "tabDocType" ( "modified_by" varchar(255) DEFAULT NULL, "owner" varchar(255) DEFAULT NULL, "docstatus" smallint NOT NULL DEFAULT 0, - "parent" varchar(255) DEFAULT NULL, - "parentfield" varchar(255) DEFAULT NULL, - "parenttype" varchar(255) DEFAULT NULL, "idx" bigint NOT NULL DEFAULT 0, "search_fields" varchar(255) DEFAULT NULL, "issingle" smallint NOT NULL DEFAULT 0, diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index a2d5be0b70..9487bc2fa7 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -5,26 +5,37 @@ from frappe.database.schema import DBTable, get_definition class PostgresTable(DBTable): def create(self): - add_text = '' + add_text = "" # columns column_defs = self.get_column_definitions() - if column_defs: add_text += ',\n'.join(column_defs) + if column_defs: + add_text += ",\n".join(column_defs) + + # child table columns + if self.meta.get("istable") or 0: + if column_defs: + add_text += ",\n" + + add_text += ",\n".join( + ( + "parent varchar({varchar_len})", + "parentfield varchar({varchar_len})", + "parenttype varchar({varchar_len})" + ) + ) # TODO: set docstatus length # create table - frappe.db.sql("""create table `%s` ( + frappe.db.sql(("""create table `%s` ( name varchar({varchar_len}) not null primary key, creation timestamp(6), modified timestamp(6), modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus smallint not null default '0', - parent varchar({varchar_len}), - parentfield varchar({varchar_len}), - parenttype varchar({varchar_len}), idx bigint not null default '0', - %s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text)) + %s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN)) self.create_indexes() frappe.db.commit() diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 9a6dd502dc..dd54385c83 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -106,6 +106,9 @@ class DBTable: columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in frappe.db.STANDARD_VARCHAR_COLUMNS] + if self.meta.get("istable"): + columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in + frappe.db.CHILD_TABLE_COLUMNS] columns += self.columns.values() for col in columns: @@ -300,11 +303,12 @@ def validate_column_length(fieldname): def get_definition(fieldtype, precision=None, length=None): d = frappe.db.type_map.get(fieldtype) - # convert int to long int if the length of the int is greater than 11 - if fieldtype == "Int" and length and length > 11: - d = frappe.db.type_map.get("Long Int") + if not d: + return - if not d: return + if fieldtype == "Int" and length and length > 11: + # convert int to long int if the length of the int is greater than 11 + d = frappe.db.type_map.get("Long Int") coltype = d[0] size = d[1] if d[1] else None @@ -315,19 +319,44 @@ def get_definition(fieldtype, precision=None, length=None): if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6: size = '21,9' - if coltype == "varchar" and length: - size = length + if length: + if coltype == "varchar": + size = length + elif coltype == "int" and length < 11: + # allow setting custom length for int if length provided is less than 11 + # NOTE: this will only be applicable for mariadb as frappe implements int + # in postgres as bigint (as seen in type_map) + size = length if size is not None: coltype = "{coltype}({size})".format(coltype=coltype, size=size) return coltype -def add_column(doctype, column_name, fieldtype, precision=None): +def add_column( + doctype, + column_name, + fieldtype, + precision=None, + length=None, + default=None, + not_null=False +): if column_name in frappe.db.get_table_columns(doctype): # already exists return frappe.db.commit() - frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype, - column_name, get_definition(fieldtype, precision))) + + query = "alter table `tab%s` add column %s %s" % ( + doctype, + column_name, + get_definition(fieldtype, precision, length) + ) + + if not_null: + query += " not null" + if default: + query += f" default '{default}'" + + frappe.db.sql(query) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index a0523d90cd..20887f8886 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -42,13 +42,13 @@ def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None): doc = frappe.get_doc(doctype, d) try: message = '' - if action == 'submit' and doc.docstatus==0: + if action == 'submit' and doc.docstatus.is_draft(): doc.submit() message = _('Submiting {0}').format(doctype) - elif action == 'cancel' and doc.docstatus==1: + elif action == 'cancel' and doc.docstatus.is_submitted(): doc.cancel() message = _('Cancelling {0}').format(doctype) - elif action == 'update' and doc.docstatus < 2: + elif action == 'update' and not doc.docstatus.is_cancelled(): doc.update(data) doc.save() message = _('Updating {0}').format(doctype) diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 381c24a765..d44c481210 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -148,8 +148,6 @@ def update_tags(doc, tags): "doctype": "Tag Link", "document_type": doc.doctype, "document_name": doc.name, - "parenttype": doc.doctype, - "parent": doc.name, "title": doc.get_title() or '', "tag": tag }).insert(ignore_permissions=True) diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index cd87c898d8..572d3f2a94 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -389,8 +389,6 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): else: return results - me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) - for dt, link in linkinfo.items(): filters = [] link["doctype"] = dt @@ -413,11 +411,16 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters")) elif link.get("get_parent"): - if me and me.parent and me.parenttype == dt: + ret = None + + # check for child table + if not frappe.get_meta(doctype).istable: + continue + + me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) + if me and me.parenttype == dt: ret = frappe.get_all(doctype=dt, fields=fields, filters=[[dt, "name", '=', me.parent]]) - else: - ret = None elif link.get("child_doctype"): or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")] @@ -473,7 +476,7 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled)) ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled)) - filters=[['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]] + filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters) @@ -498,12 +501,12 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): - filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]] + filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1) - links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1) + links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1) ret = {} @@ -529,34 +532,37 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False): ret = {} - filters=[['fieldtype','=', 'Dynamic Link']] + filters = [['fieldtype','=', 'Dynamic Link']] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find dynamic links of parents links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) - links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) + links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) for df in links: if is_single(df.doctype): continue - # optimized to get both link exists and parenttype - possible_link = frappe.get_all(df.doctype, filters={df.doctype_fieldname: doctype}, - fields=['parenttype'], distinct=True) + is_child = frappe.get_meta(df.doctype).istable + possible_link = frappe.get_all( + df.doctype, + filters={df.doctype_fieldname: doctype}, + fields=["parenttype"] if is_child else None, + distinct=True + ) if not possible_link: continue - for d in possible_link: - # is child - if d.parenttype: + if is_child: + for d in possible_link: ret[d.parenttype] = { "child_doctype": df.doctype, "fieldname": [df.fieldname], "doctype_fieldname": df.doctype_fieldname } - else: - ret[df.doctype] = { - "fieldname": [df.fieldname], - "doctype_fieldname": df.doctype_fieldname - } + else: + ret[df.doctype] = { + "fieldname": [df.fieldname], + "doctype_fieldname": df.doctype_fieldname + } return ret diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 38b671d629..58d5b30103 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -91,8 +91,8 @@ def get_docinfo(doc=None, doctype=None, name=None): raise frappe.PermissionError all_communications = _get_communications(doc.doctype, doc.name) - automated_messages = filter(lambda x: x['communication_type'] == 'Automated Message', all_communications) - communications_except_auto_messages = filter(lambda x: x['communication_type'] != 'Automated Message', all_communications) + automated_messages = [msg for msg in all_communications if msg['communication_type'] == 'Automated Message'] + communications_except_auto_messages = [msg for msg in all_communications if msg['communication_type'] != 'Automated Message'] docinfo = frappe._dict(user_info = {}) @@ -119,6 +119,7 @@ def get_docinfo(doc=None, doctype=None, name=None): update_user_info(docinfo) frappe.response["docinfo"] = docinfo + return docinfo def add_comments(doc, docinfo): # divide comments into separate lists diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 27ac882016..c45fc9bfdd 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -6,7 +6,7 @@ import frappe, json import frappe.permissions from frappe.model.db_query import DatabaseQuery -from frappe.model import default_fields, optional_fields +from frappe.model import default_fields, optional_fields, child_table_fields from frappe import _ from io import StringIO from frappe.core.doctype.access_log.access_log import make_access_log @@ -156,7 +156,7 @@ def raise_invalid_field(fieldname): def is_standard(fieldname): if '.' in fieldname: parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None) - return fieldname in default_fields or fieldname in optional_fields + return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields def extract_fieldname(field): for text in (',', '/*', '#'): @@ -319,7 +319,7 @@ def export_query(): if add_totals_row: ret = append_totals_row(ret) - data = [['Sr'] + get_labels(db_query.fields, doctype)] + data = [[_('Sr')] + get_labels(db_query.fields, doctype)] for i, row in enumerate(ret): data.append([i+1] + list(row)) @@ -378,7 +378,8 @@ def get_labels(fields, doctype): for key in fields: key = key.split(" as ")[0] - if key.startswith(('count(', 'sum(', 'avg(')): continue + if key.startswith(('count(', 'sum(', 'avg(')): + continue if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") @@ -386,10 +387,16 @@ def get_labels(fields, doctype): parenttype = doctype fieldname = fieldname.strip("`") - df = frappe.get_meta(parenttype).get_field(fieldname) - label = df.label if df else fieldname.title() - if label in labels: - label = doctype + ": " + label + if parenttype == doctype and fieldname == "name": + label = _("ID", context="Label of name column in report") + else: + df = frappe.get_meta(parenttype).get_field(fieldname) + label = _(df.label if df else fieldname.title()) + if parenttype != doctype: + # If the column is from a child table, append the child doctype. + # For example, "Item Code (Sales Invoice Item)". + label += f" ({ _(parenttype) })" + labels.append(label) return labels diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index 34728375cd..682f0df7cf 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -252,7 +252,7 @@ def make_links(columns, data): if col.options and row.get(col.options): row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname]) elif col.fieldtype == "Currency": - doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.parent else None + doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.get("parent") else None # Pass the Document to get the currency based on docfield option row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc) return columns, data diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index a05e20da24..3a1b683398 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -421,10 +421,10 @@ class EmailAccount(Document): def get_failed_attempts_count(self): return cint(frappe.cache().get('{0}:email-account-failed-attempts'.format(self.name))) - def receive(self, test_mails=None): + def receive(self): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" exceptions = [] - inbound_mails = self.get_inbound_mails(test_mails=test_mails) + inbound_mails = self.get_inbound_mails() for mail in inbound_mails: try: communication = mail.process() @@ -457,20 +457,19 @@ class EmailAccount(Document): if exceptions: raise Exception(frappe.as_json(exceptions)) - def get_inbound_mails(self, test_mails=None) -> List[InboundMail]: + def get_inbound_mails(self) -> List[InboundMail]: """retrive and return inbound mails. """ mails = [] - def process_mail(messages): + def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages['uid_list'][index] if messages.get('uid_list') else None - seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0 - mails.append(InboundMail(message, self, uid, seen_status)) - - if frappe.local.flags.in_test: - return [InboundMail(msg, self) for msg in test_mails or []] + seen_status = messages.get('seen_status', {}).get(uid) + if self.email_sync_option != 'UNSEEN' or seen_status != "SEEN": + # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' + mails.append(InboundMail(message, self, uid, seen_status, append_to)) if not self.enable_incoming: return [] @@ -481,10 +480,10 @@ class EmailAccount(Document): if self.use_imap: # process all given imap folder for folder in self.imap_folder: - email_server.select_imap_folder(folder.folder_name) - email_server.settings['uid_validity'] = folder.uidvalidity - messages = email_server.get_messages(folder=folder.folder_name) or {} - process_mail(messages) + if email_server.select_imap_folder(folder.folder_name): + email_server.settings['uid_validity'] = folder.uidvalidity + messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {} + process_mail(messages, folder.append_to) else: # process the pop3 account messages = email_server.get_messages() or {} @@ -494,7 +493,6 @@ class EmailAccount(Document): except Exception: frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) return [] - return mails def handle_bad_emails(self, uid, raw, reason): diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 6d26f9f070..f609c2947d 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -14,11 +14,11 @@ from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments from frappe.email.doctype.email_account.email_account import notify_unreplied +from unittest.mock import patch + make_test_records("User") make_test_records("Email Account") - - class TestEmailAccount(unittest.TestCase): @classmethod def setUpClass(cls): @@ -45,10 +45,21 @@ class TestEmailAccount(unittest.TestCase): def test_incoming(self): cleanup("test_sender@example.com") - test_mails = [self.get_test_mail('incoming-1.raw')] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-1.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) @@ -72,11 +83,21 @@ class TestEmailAccount(unittest.TestCase): existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile: - test_mails = [testfile.read()] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-2.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) @@ -93,11 +114,21 @@ class TestEmailAccount(unittest.TestCase): def test_incoming_attached_email_from_outlook_plain_text_only(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-3.raw"), "r") as f: - test_mails = [f.read()] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-3.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) @@ -106,11 +137,21 @@ class TestEmailAccount(unittest.TestCase): def test_incoming_attached_email_from_outlook_layers(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-4.raw"), "r") as f: - test_mails = [f.read()] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-4.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) @@ -151,11 +192,23 @@ class TestEmailAccount(unittest.TestCase): with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw"), "r") as f: raw = f.read() raw = raw.replace("<-- in-reply-to -->", sent_mail.get("Message-Id")) - test_mails = [raw] # parse reply + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + raw + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) sent = frappe.get_doc("Communication", sent_name) @@ -173,8 +226,20 @@ class TestEmailAccount(unittest.TestCase): test_mails.append(f.read()) # parse reply + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': test_mails, + 'seen_status': { + 2: 'UNSEEN', + 3: 'UNSEEN' + }, + 'uid_list': [2, 3] + } + } + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) @@ -197,11 +262,22 @@ class TestEmailAccount(unittest.TestCase): # get test mail with message-id as in-reply-to with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f: - test_mails = [f.read().replace('{{ message_id }}', last_mail.message_id)] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + f.read().replace('{{ message_id }}', last_mail.message_id) + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } # pull the mail email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) @@ -213,10 +289,21 @@ class TestEmailAccount(unittest.TestCase): def test_auto_reply(self): cleanup("test_sender@example.com") - test_mails = [self.get_test_mail('incoming-1.raw')] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-1.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, @@ -246,6 +333,91 @@ class TestEmailAccount(unittest.TestCase): with self.assertRaises(Exception): email_account.validate() + def test_append_to(self): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + mail_content = self.get_test_mail(fname="incoming-2.raw") + + inbound_mail = InboundMail(mail_content, email_account, 12345, 1, 'ToDo') + communication = inbound_mail.process() + # the append_to for the email is set to ToDO in "_Test Email Account 1" + self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertTrue(communication.reference_name) + self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) + + def test_append_to_with_imap_folders(self): + mail_content_1 = self.get_test_mail(fname="incoming-1.raw") + mail_content_2 = self.get_test_mail(fname="incoming-2.raw") + mail_content_3 = self.get_test_mail(fname="incoming-3.raw") + + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + mail_content_1, + mail_content_2 + ], + 'seen_status': { + 0: 'UNSEEN', + 1: 'UNSEEN' + }, + 'uid_list': [0,1] + }, + # append_to = Communication + '"Test Folder"': { + 'latest_messages': [ + mail_content_3 + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages) + self.assertEqual(len(mails), 3) + + inbox_mails = 0 + test_folder_mails = 0 + + for mail in mails: + communication = mail.process() + if mail.append_to == 'ToDo': + inbox_mails += 1 + self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertTrue(communication.reference_name) + self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) + else: + test_folder_mails += 1 + self.assertEqual(communication.reference_doctype, None) + + self.assertEqual(inbox_mails, 2) + self.assertEqual(test_folder_mails, 1) + + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) + @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) + def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + from frappe.email.receive import EmailServer + + def get_mocked_messages(**kwargs): + return messages.get(kwargs["folder"], {}) + + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): + mails = email_account.get_inbound_mails() + + return mails + + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) + @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) + def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + def get_mocked_messages(**kwargs): + return messages.get(kwargs["folder"], {}) + + from frappe.email.receive import EmailServer + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): + email_account.receive() + class TestInboundMail(unittest.TestCase): @classmethod def setUpClass(cls): @@ -313,11 +485,11 @@ class TestInboundMail(unittest.TestCase): email_account = frappe.get_doc("Email Account", "_Test Email Account 1") inbound_mail = InboundMail(mail_content, email_account, 12345, 1) - new_communiction = inbound_mail.process() + new_communication = inbound_mail.process() # Make sure that uid is changed to new uid - self.assertEqual(new_communiction.uid, 12345) - self.assertEqual(communication.name, new_communiction.name) + self.assertEqual(new_communication.uid, 12345) + self.assertEqual(communication.name, new_communication.name) def test_find_parent_email_queue(self): """If the mail is reply to the already sent mail, there will be a email queue record. diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json index 450895d7a6..66eb5a9b2e 100644 --- a/frappe/email/doctype/email_account/test_records.json +++ b/frappe/email/doctype/email_account/test_records.json @@ -20,7 +20,7 @@ "pop3_server": "pop.test.example.com", "no_remaining":"0", "append_to": "ToDo", - "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], + "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}], "track_email_status": 1 }, { diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 4da83bd0d2..9b4f3b984c 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -479,21 +479,24 @@ class QueueBuilder: EmailUnsubscribe = DocType("Email Unsubscribe") - unsubscribed = ( - frappe.qb.from_(EmailUnsubscribe).select( - EmailUnsubscribe.email - ).where( - EmailUnsubscribe.email.isin(all_ids) - & ( - ( - (EmailUnsubscribe.reference_doctype == self.reference_doctype) - & (EmailUnsubscribe.reference_name == self.reference_name) - ) | ( - EmailUnsubscribe.global_unsubscribe == 1 + if len(all_ids) > 0: + unsubscribed = ( + frappe.qb.from_(EmailUnsubscribe).select( + EmailUnsubscribe.email + ).where( + EmailUnsubscribe.email.isin(all_ids) + & ( + ( + (EmailUnsubscribe.reference_doctype == self.reference_doctype) + & (EmailUnsubscribe.reference_name == self.reference_name) + ) | ( + EmailUnsubscribe.global_unsubscribe == 1 + ) ) - ) - ).distinct() - ).run(pluck=True) + ).distinct() + ).run(pluck=True) + else: + unsubscribed = None self._unsubscribed_user_emails = unsubscribed or [] return self._unsubscribed_user_emails diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 77979f9735..3fd96bdb6b 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -137,7 +137,7 @@ def get_context(context): if self.set_property_after_alert: allow_update = True - if doc.docstatus == 1 and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit: + if doc.docstatus.is_submitted() and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit: allow_update = False try: if allow_update and not doc.flags.in_notification_update: diff --git a/frappe/email/receive.py b/frappe/email/receive.py index dd64d0df80..b8156d5d9b 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -108,7 +108,8 @@ class EmailServer: raise def select_imap_folder(self, folder): - self.imap.select(folder) + res = self.imap.select(f'"{folder}"') + return res[0] == 'OK' # The folder exsits TODO: handle other resoponses too def logout(self): if cint(self.settings.use_imap): @@ -582,10 +583,11 @@ class Email: class InboundMail(Email): """Class representation of incoming mail along with mail handlers. """ - def __init__(self, content, email_account, uid=None, seen_status=None): + def __init__(self, content, email_account, uid=None, seen_status=None, append_to=None): super().__init__(content) self.email_account = email_account self.uid = uid or -1 + self.append_to = append_to self.seen_status = seen_status or 0 # System documents related to this mail @@ -623,15 +625,18 @@ class InboundMail(Email): if self.parent_communication(): data['in_reply_to'] = self.parent_communication().name + append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to + if self.reference_document(): data['reference_doctype'] = self.reference_document().doctype data['reference_name'] = self.reference_document().name - elif self.email_account.append_to and self.email_account.append_to != 'Communication': - reference_doc = self._create_reference_document(self.email_account.append_to) - if reference_doc: - data['reference_doctype'] = reference_doc.doctype - data['reference_name'] = reference_doc.name - data['is_first'] = True + else: + if append_to and append_to != 'Communication': + reference_doc = self._create_reference_document(append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True if self.is_notification(): # Disable notifications for notification. diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py index 8f1e5504da..0565b3219d 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py @@ -5,7 +5,7 @@ import frappe import json from frappe import _ from frappe.model.document import Document -from frappe.model import default_fields +from frappe.model import default_fields, child_table_fields class DocumentTypeMapping(Document): def validate(self): @@ -14,7 +14,7 @@ class DocumentTypeMapping(Document): def validate_inner_mapping(self): meta = frappe.get_meta(self.local_doctype) for field_map in self.field_mapping: - if field_map.local_fieldname not in default_fields: + if field_map.local_fieldname not in (default_fields + child_table_fields): field = meta.get_field(field_map.local_fieldname) if not field: frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx)) diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 8449425bc1..6ee72b5f81 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -103,6 +103,7 @@ class DocumentAlreadyRestored(ValidationError): pass class AttachmentLimitReached(ValidationError): pass class QueryTimeoutError(Exception): pass class QueryDeadlockError(Exception): pass +class TooManyWritesError(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index b50a0304a5..be9496c85b 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -90,11 +90,14 @@ default_fields = ( 'creation', 'modified', 'modified_by', + 'docstatus', + 'idx' +) + +child_table_fields = ( 'parent', 'parentfield', - 'parenttype', - 'idx', - 'docstatus' + 'parenttype' ) optional_fields = ( diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 11e97a38b9..307d95e84b 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -1,9 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE + import frappe import datetime from frappe import _ -from frappe.model import default_fields, table_fields +from frappe.model import default_fields, table_fields, child_table_fields from frappe.model.naming import set_new_name from frappe.model.utils.link_count import notify_link_count from frappe.modules import load_doctype_module @@ -11,6 +12,7 @@ from frappe.model import display_fieldtypes from frappe.utils import (cint, flt, now, cstr, strip_html, sanitize_html, sanitize_email, cast_fieldtype) from frappe.utils.html_utils import unescape_html +from frappe.model.docstatus import DocStatus max_positive_value = { 'smallint': 2 ** 15, @@ -20,6 +22,7 @@ max_positive_value = { DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link') + def get_controller(doctype): """Returns the **class** object of the given DocType. For `custom` type, returns `frappe.model.document.Document`. @@ -101,6 +104,10 @@ class BaseDocument(object): "balance": 42000 }) """ + + # QUESTION: why do we need the 1st for loop? + # we're essentially setting the values in d, in the 2nd for loop (?) + # first set default field values of base document for key in default_fields: if key in d: @@ -205,7 +212,10 @@ class BaseDocument(object): raise ValueError def remove(self, doc): - self.get(doc.parentfield).remove(doc) + # Usage: from the parent doc, pass the child table doc + # to remove that child doc from the child table, thus removing it from the parent doc + if doc.get("parentfield"): + self.get(doc.parentfield).remove(doc) def _init_child(self, value, key): if not self.doctype: @@ -224,7 +234,7 @@ class BaseDocument(object): value.parentfield = key if value.docstatus is None: - value.docstatus = 0 + value.docstatus = DocStatus.draft() if not getattr(value, "idx", None): value.idx = len(self.get(key) or []) + 1 @@ -282,8 +292,11 @@ class BaseDocument(object): if key not in self.__dict__: self.__dict__[key] = None - if key in ("idx", "docstatus") and self.__dict__[key] is None: - self.__dict__[key] = 0 + if self.__dict__[key] is None: + if key == "docstatus": + self.docstatus = DocStatus.draft() + elif key == "idx": + self.__dict__[key] = 0 for key in self.get_valid_columns(): if key not in self.__dict__: @@ -304,12 +317,27 @@ class BaseDocument(object): def is_new(self): return self.get("__islocal") - def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False): + @property + def docstatus(self): + return DocStatus(self.get("docstatus")) + + @docstatus.setter + def docstatus(self, value): + self.__dict__["docstatus"] = DocStatus(cint(value)) + + def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False): doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str) doc["doctype"] = self.doctype for df in self.meta.get_table_fields(): children = self.get(df.fieldname) or [] - doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children] + doc[df.fieldname] = [ + d.as_dict( + convert_dates_to_str=convert_dates_to_str, + no_nulls=no_nulls, + no_default_fields=no_default_fields, + no_child_table_fields=no_child_table_fields + ) for d in children + ] if no_nulls: for k in list(doc): @@ -321,6 +349,11 @@ class BaseDocument(object): if k in default_fields: del doc[k] + if no_child_table_fields: + for k in list(doc): + if k in child_table_fields: + del doc[k] + for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"): if self.get(key): doc[key] = self.get(key) @@ -492,7 +525,7 @@ class BaseDocument(object): self.set(df.fieldname, flt(self.get(df.fieldname))) if self.docstatus is not None: - self.docstatus = cint(self.docstatus) + self.docstatus = DocStatus(cint(self.docstatus)) def _get_missing_mandatory_fields(self): """Get mandatory fields that do not have any values""" @@ -500,12 +533,12 @@ class BaseDocument(object): if df.fieldtype in table_fields: return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label)) - elif self.parentfield: + # check if parentfield exists (only applicable for child table doctype) + elif self.get("parentfield"): return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)), _("Row"), self.idx, _("Value missing for"), _(df.label)) - else: - return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label)) + return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label)) missing = [] @@ -524,10 +557,11 @@ class BaseDocument(object): def get_invalid_links(self, is_submittable=False): """Returns list of invalid links and also updates fetch values if not set""" def get_msg(df, docname): - if self.parentfield: + # check if parentfield exists (only applicable for child table doctype) + if self.get("parentfield"): return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname) - else: - return "{}: {}".format(_(df.label), docname) + + return "{}: {}".format(_(df.label), docname) invalid_links = [] cancelled_links = [] @@ -581,7 +615,7 @@ class BaseDocument(object): setattr(self, df.fieldname, values.name) for _df in fields_to_fetch: - if self.is_new() or self.docstatus != 1 or _df.allow_on_submit: + if self.is_new() or not self.docstatus.is_submitted() or _df.allow_on_submit: self.set_fetch_from_value(doctype, _df, values) notify_link_count(doctype, docname) @@ -591,7 +625,7 @@ class BaseDocument(object): elif (df.fieldname != "amended_from" and (is_submittable or self.meta.is_submittable) and frappe.get_meta(doctype).is_submittable - and cint(frappe.db.get_value(doctype, docname, "docstatus"))==2): + and cint(frappe.db.get_value(doctype, docname, "docstatus")) == DocStatus.cancelled()): cancelled_links.append((df.fieldname, docname, get_msg(df, docname))) @@ -601,11 +635,8 @@ class BaseDocument(object): fetch_from_fieldname = df.fetch_from.split('.')[-1] value = values[fetch_from_fieldname] if df.fieldtype in ['Small Text', 'Text', 'Data']: - if fetch_from_fieldname in default_fields: - from frappe.model.meta import get_default_df - fetch_from_df = get_default_df(fetch_from_fieldname) - else: - fetch_from_df = frappe.get_meta(doctype).get_field(fetch_from_fieldname) + from frappe.model.meta import get_default_df + fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname) if not fetch_from_df: frappe.throw( @@ -740,9 +771,9 @@ class BaseDocument(object): def throw_length_exceeded_error(self, df, max_length, value): - if self.parentfield and self.idx: + # check if parentfield exists (only applicable for child table doctype) + if self.get("parentfield"): reference = _("{0}, Row {1}").format(_(self.doctype), self.idx) - else: reference = "{0} {1}".format(_(self.doctype), self.name) @@ -805,8 +836,8 @@ class BaseDocument(object): or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode", "Code") # cancelled and submit but not update after submit should be ignored - or self.docstatus==2 - or (self.docstatus==1 and not df.get("allow_on_submit"))): + or self.docstatus.is_cancelled() + or (self.docstatus.is_submitted() and not df.get("allow_on_submit"))): continue else: @@ -853,7 +884,7 @@ class BaseDocument(object): :param parentfield: If fieldname is in child table.""" from frappe.model.meta import get_field_precision - if parentfield and not isinstance(parentfield, str): + if parentfield and not isinstance(parentfield, str) and parentfield.get("parentfield"): parentfield = parentfield.parentfield cache_key = parentfield or "main" @@ -880,7 +911,7 @@ class BaseDocument(object): from frappe.utils.formatters import format_value df = self.meta.get_field(fieldname) - if not df and fieldname in default_fields: + if not df: from frappe.model.meta import get_default_df df = get_default_df(fieldname) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 2fddcf9e33..2cc99575d6 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -212,7 +212,7 @@ def check_permission_and_not_submitted(doc): .format(doc.doctype, doc.name), raise_exception=frappe.PermissionError) # check if submitted - if doc.docstatus == 1: + if doc.docstatus.is_submitted(): frappe.msgprint(_("{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.").format(_(doc.doctype), doc.name, "", ""), raise_exception=True) @@ -222,32 +222,35 @@ def check_if_doc_is_linked(doc, method="Delete"): """ from frappe.model.rename_doc import get_link_fields link_fields = get_link_fields(doc.doctype) - link_fields = [[lf['parent'], lf['fieldname'], lf['issingle']] for lf in link_fields] + ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + + for lf in link_fields: + link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle'] - for link_dt, link_field, issingle in link_fields: if not issingle: - for item in frappe.db.get_values(link_dt, {link_field:doc.name}, - ["name", "parent", "parenttype", "docstatus"], as_dict=True): - linked_doctype = item.parenttype if item.parent else link_dt + fields = ["name", "docstatus"] + if frappe.get_meta(link_dt).istable: + fields.extend(["parent", "parenttype"]) - ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True): + # available only in child table cases + item_parent = getattr(item, "parent", None) + linked_doctype = item.parenttype if item_parent else link_dt if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'): # don't check for communication and todo! continue - if not item: - continue - elif method != "Delete" and (method != "Cancel" or item.docstatus != 1): + if method != "Delete" and (method != "Cancel" or item.docstatus != 1): # don't raise exception if not # linked to a non-cancelled doc when deleting or to a submitted doc when cancelling continue - elif link_dt == doc.doctype and (item.parent or item.name) == doc.name: + elif link_dt == doc.doctype and (item_parent or item.name) == doc.name: # don't raise exception if not # linked to same item or doc having same name as the item continue else: - reference_docname = item.parent or item.name + reference_docname = item_parent or item.name raise_link_exists_exception(doc, linked_doctype, reference_docname) else: diff --git a/frappe/model/docstatus.py b/frappe/model/docstatus.py new file mode 100644 index 0000000000..01aab1e491 --- /dev/null +++ b/frappe/model/docstatus.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. See LICENSE + + +class DocStatus(int): + def is_draft(self): + return self == self.draft() + + def is_submitted(self): + return self == self.submitted() + + def is_cancelled(self): + return self == self.cancelled() + + @classmethod + def draft(cls): + return cls(0) + + @classmethod + def submitted(cls): + return cls(1) + + @classmethod + def cancelled(cls): + return cls(2) diff --git a/frappe/model/document.py b/frappe/model/document.py index 1217b45aaf..f7ba9250fa 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,13 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe +import hashlib +import json import time +from werkzeug.exceptions import NotFound + +import frappe from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc -from werkzeug.exceptions import NotFound, Forbidden -import hashlib, json +from frappe.model.naming import set_new_name +from frappe.model.docstatus import DocStatus from frappe.model import optional_fields, table_fields from frappe.model.workflow import validate_workflow from frappe.model.workflow import set_workflow_state_on_action @@ -17,6 +20,7 @@ from frappe.desk.form.document_follow import follow_document from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event from frappe.utils.data import get_absolute_url + # once_only validation # methods @@ -307,9 +311,6 @@ class Document(BaseDocument): self.check_permission("write", "save") - if self.docstatus == 2: - self._rename_doc_on_cancel() - self.set_user_and_timestamp() self.set_docstatus() self.check_if_latest() @@ -490,7 +491,7 @@ class Document(BaseDocument): def set_docstatus(self): if self.docstatus is None: - self.docstatus=0 + self.docstatus = DocStatus.draft() for d in self.get_all_children(): d.docstatus = self.docstatus @@ -526,7 +527,7 @@ class Document(BaseDocument): def _validate_non_negative(self): def get_msg(df): - if self.parentfield: + if self.get("parentfield"): return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)), _("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label))) else: @@ -720,6 +721,7 @@ class Document(BaseDocument): else: tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` where name = %s for update""".format(self.doctype), self.name, as_dict=True) + if not tmp: frappe.throw(_("Record does not exist")) else: @@ -740,7 +742,7 @@ class Document(BaseDocument): else: self.check_docstatus_transition(0) - def check_docstatus_transition(self, docstatus): + def check_docstatus_transition(self, to_docstatus): """Ensures valid `docstatus` transition. Valid transitions are (number in brackets is `docstatus`): @@ -751,31 +753,32 @@ class Document(BaseDocument): """ if not self.docstatus: - self.docstatus = 0 - if docstatus==0: - if self.docstatus==0: + self.docstatus = DocStatus.draft() + + if to_docstatus == DocStatus.draft(): + if self.docstatus.is_draft(): self._action = "save" - elif self.docstatus==1: + elif self.docstatus.is_submitted(): self._action = "submit" self.check_permission("submit") - elif self.docstatus==2: + elif self.docstatus.is_cancelled(): raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)")) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) - elif docstatus==1: - if self.docstatus==1: + elif to_docstatus == DocStatus.submitted(): + if self.docstatus.is_submitted(): self._action = "update_after_submit" self.check_permission("submit") - elif self.docstatus==2: + elif self.docstatus.is_cancelled(): self._action = "cancel" self.check_permission("cancel") - elif self.docstatus==0: + elif self.docstatus.is_draft(): raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 (Submitted) to 0 (Draft)")) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) - elif docstatus==2: + elif to_docstatus == DocStatus.cancelled(): raise frappe.ValidationError(_("Cannot edit cancelled document")) def set_parent_in_children(self): @@ -929,14 +932,14 @@ class Document(BaseDocument): @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" - self.docstatus = 1 + self.docstatus = DocStatus.submitted() return self.save() @whitelist.__func__ def _cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves. """ - self.docstatus = 2 + self.docstatus = DocStatus.cancelled() return self.save() @whitelist.__func__ @@ -954,7 +957,7 @@ class Document(BaseDocument): frappe.delete_doc(self.doctype, self.name, ignore_permissions = ignore_permissions, flags=self.flags) def run_before_save_methods(self): - """Run standard methods before `INSERT` or `UPDATE`. Standard Methods are: + """Run standard methods before `INSERT` or `UPDATE`. Standard Methods are: - `validate`, `before_save` for **Save**. - `validate`, `before_submit` for **Submit**. @@ -1199,7 +1202,7 @@ class Document(BaseDocument): if not frappe.compare(val1, condition, val2): label = doc.meta.get_label(fieldname) condition_str = error_condition_map.get(condition, condition) - if doc.parentfield: + if doc.get("parentfield"): msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2) else: msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2) @@ -1223,7 +1226,7 @@ class Document(BaseDocument): doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]})) for fieldname in fieldnames: - doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield))) + doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield")))) def get_url(self): """Returns Desk URL for this document.""" @@ -1371,19 +1374,16 @@ class Document(BaseDocument): from frappe.desk.doctype.tag.tag import DocTags return DocTags(self.doctype).get_tags(self.name).split(",")[1:] - def _rename_doc_on_cancel(self): - new_name = gen_new_name_for_cancelled_doc(self) - frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) - self.name = new_name - def __repr__(self): name = self.name or "unsaved" doctype = self.__class__.__name__ docstatus = f" docstatus={self.docstatus}" if self.docstatus else "" - parent = f" parent={self.parent}" if self.parent else "" + repr_str = f"<{doctype}: {name}{docstatus}" - return f"<{doctype}: {name}{docstatus}{parent}>" + if not hasattr(self, "parent"): + return repr_str + ">" + return f"{repr_str} parent={self.parent}>" def __str__(self): name = self.name or "unsaved" diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index bde4fb6d73..f40a43bb73 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -4,7 +4,7 @@ import json import frappe from frappe import _ -from frappe.model import default_fields, table_fields +from frappe.model import default_fields, table_fields, child_table_fields from frappe.utils import cstr @@ -149,6 +149,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent): no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] + [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] + list(default_fields) + + list(child_table_fields) + list(table_map.get("field_no_map", []))) for df in target_doc.meta.get("fields"): diff --git a/frappe/model/meta.py b/frappe/model/meta.py index a483f3f2d6..372392f689 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -18,7 +18,7 @@ from datetime import datetime import click import frappe, json, os from frappe.utils import cstr, cint, cast -from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields +from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields, child_table_fields from frappe.model.document import Document from frappe.model.base_document import BaseDocument from frappe.modules import load_doctype_module @@ -191,6 +191,8 @@ class Meta(Document): else: self._valid_columns = self.default_fields + \ [df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes] + if self.istable: + self._valid_columns += list(child_table_fields) return self._valid_columns @@ -520,7 +522,7 @@ class Meta(Document): '''add `links` child table in standard link dashboard format''' dashboard_links = [] - if hasattr(self, 'links') and self.links: + if getattr(self, 'links', None): dashboard_links.extend(self.links) if not data.transactions: @@ -625,9 +627,9 @@ def get_field_currency(df, doc=None): frappe.local.field_currency = frappe._dict() if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or - (doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))): + (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))): - ref_docname = doc.parent or doc.name + ref_docname = doc.get("parent") or doc.name if ":" in cstr(df.get("options")): split_opts = df.get("options").split(":") @@ -635,7 +637,7 @@ def get_field_currency(df, doc=None): currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2]) else: currency = doc.get(df.get("options")) - if doc.parent: + if doc.get("parenttype"): if currency: ref_docname = doc.name else: @@ -648,7 +650,7 @@ def get_field_currency(df, doc=None): .setdefault(df.fieldname, currency) return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \ - (doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname)) + (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname)) def get_field_precision(df, doc=None, currency=None): """get precision based on DocField options and fieldvalue in doc""" @@ -669,19 +671,25 @@ def get_field_precision(df, doc=None, currency=None): def get_default_df(fieldname): - if fieldname in default_fields: + if fieldname in (default_fields + child_table_fields): if fieldname in ("creation", "modified"): return frappe._dict( fieldname = fieldname, fieldtype = "Datetime" ) - else: + elif fieldname in ("idx", "docstatus"): return frappe._dict( fieldname = fieldname, - fieldtype = "Data" + fieldtype = "Int" ) + return frappe._dict( + fieldname = fieldname, + fieldtype = "Data" + ) + + def trim_tables(doctype=None, dry_run=False, quiet=False): """ Removes database fields that don't exist in the doctype (json or custom field). This may be needed @@ -713,7 +721,7 @@ def trim_tables(doctype=None, dry_run=False, quiet=False): def trim_table(doctype, dry_run=True): frappe.cache().hdel('table_columns', f"tab{doctype}") - ignore_fields = default_fields + optional_fields + ignore_fields = default_fields + optional_fields + child_table_fields columns = frappe.db.get_table_columns(doctype) fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value() is_internal = lambda f: f not in ignore_fields and not f.startswith("_") diff --git a/frappe/model/naming.py b/frappe/model/naming.py index f3d68f3715..b2d11a4cfc 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,14 +1,3 @@ -"""utilities to generate a document name based on various rules defined. - -NOTE: -Till version 13, whenever a submittable document is amended it's name is set to orig_name-X, -where X is a counter and it increments when amended again and so on. - -From Version 14, The naming pattern is changed in a way that amended documents will -have the original name `orig_name` instead of `orig_name-X`. To make this happen -the cancelled document naming pattern is changed to 'orig_name-CANC-X'. -""" - # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE @@ -40,7 +29,7 @@ def set_new_name(doc): doc.name = None if getattr(doc, "amended_from", None): - doc.name = _get_amended_name(doc) + _set_amended_name(doc) return elif getattr(doc.meta, "issingle", False): @@ -256,18 +245,6 @@ def revert_series_if_last(key, name, doc=None): * prefix = #### and hashes = 2021 (hash doesn't exist) * will search hash in key then accordingly get prefix = "" """ - if hasattr(doc, 'amended_from'): - # Do not revert the series if the document is amended. - if doc.amended_from: - return - - # Get document name by parsing incase of fist cancelled document - if doc.docstatus == 2 and not doc.amended_from: - if doc.name.endswith('-CANC'): - name, _ = NameParser.parse_docname(doc.name, sep='-CANC') - else: - name, _ = NameParser.parse_docname(doc.name, sep='-CANC-') - if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: @@ -356,9 +333,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" return value -def _get_amended_name(doc): - name, _ = NameParser(doc).parse_amended_from() - return name +def _set_amended_name(doc): + am_id = 1 + am_prefix = doc.amended_from + if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): + am_id = cint(doc.amended_from.split("-")[-1]) + 1 + am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen + + doc.name = am_prefix + "-" + str(am_id) + return doc.name + def _field_autoname(autoname, doc, skip_slicing=None): """ @@ -399,83 +383,3 @@ def _format_autoname(autoname, doc): name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) return name - -class NameParser: - """Parse document name and return parts of it. - - NOTE: It handles cancellend and amended doc parsing for now. It can be expanded. - """ - def __init__(self, doc): - self.doc = doc - - def parse_amended_from(self): - """ - Cancelled document naming will be in one of these formats - - * original_name-X-CANC - This is introduced to migrate old style naming to new style - * original_name-CANC - This is introduced to migrate old style naming to new style - * original_name-CANC-X - This is the new style naming - - New style naming: In new style naming amended documents will have original name. That says, - when a document gets cancelled we need rename the document by adding `-CANC-X` to the end - so that amended documents can use the original name. - - Old style naming: cancelled documents stay with original name and when amended, amended one - gets a new name as `original_name-X`. To bring new style naming we had to change the existing - cancelled document names and that is done by adding `-CANC` to cancelled documents through patch. - """ - if not getattr(self.doc, 'amended_from', None): - return (None, None) - - # Handle old style cancelled documents (original_name-X-CANC, original_name-CANC) - if self.doc.amended_from.endswith('-CANC'): - name, _ = self.parse_docname(self.doc.amended_from, '-CANC') - amended_from_doc = frappe.get_all( - self.doc.doctype, - filters = {'name': self.doc.amended_from}, - fields = ['amended_from'], - limit=1) - - # Handle format original_name-X-CANC. - if amended_from_doc and amended_from_doc[0].amended_from: - return self.parse_docname(name, '-') - return name, None - - # Handle new style cancelled documents - return self.parse_docname(self.doc.amended_from, '-CANC-') - - @classmethod - def parse_docname(cls, name, sep='-'): - split_list = name.rsplit(sep, 1) - - if len(split_list) == 1: - return (name, None) - return (split_list[0], split_list[1]) - -def get_cancelled_doc_latest_counter(tname, docname): - """Get the latest counter used for cancelled docs of given docname. - """ - name_prefix = f'{docname}-CANC-' - - rows = frappe.db.sql(""" - select - name - from `tab{tname}` - where - name like %(name_prefix)s and docstatus=2 - """.format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1) - - if not rows: - return -1 - return max([int(row.name.replace(name_prefix, '') or -1) for row in rows]) - -def gen_new_name_for_cancelled_doc(doc): - """Generate a new name for cancelled document. - """ - if getattr(doc, "amended_from", None): - name, _ = NameParser(doc).parse_amended_from() - else: - name = doc.name - - counter = get_cancelled_doc_latest_counter(doc.doctype, name) - return f'{name}-CANC-{counter+1}' diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index e74d88c0f2..1b26cc2c3a 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -1,10 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json import frappe -from frappe.utils import cint from frappe import _ -import json +from frappe.utils import cint +from frappe.model.docstatus import DocStatus class WorkflowStateError(frappe.ValidationError): pass class WorkflowTransitionError(frappe.ValidationError): pass @@ -102,13 +103,13 @@ def apply_workflow(doc, action): doc.set(next_state.update_field, next_state.update_value) new_docstatus = cint(next_state.doc_status) - if doc.docstatus == 0 and new_docstatus == 0: + if doc.docstatus.is_draft() and new_docstatus == DocStatus.draft(): doc.save() - elif doc.docstatus == 0 and new_docstatus == 1: + elif doc.docstatus.is_draft() and new_docstatus == DocStatus.submitted(): doc.submit() - elif doc.docstatus == 1 and new_docstatus == 1: + elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.submitted(): doc.save() - elif doc.docstatus == 1 and new_docstatus == 2: + elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.cancelled(): doc.cancel() else: frappe.throw(_('Illegal Document Status for {0}').format(next_state.state)) @@ -212,10 +213,10 @@ def bulk_workflow_approval(docnames, doctype, action): frappe.db.commit() except Exception as e: if not frappe.message_log: - # Exception is raised manually and not from msgprint or throw + # Exception is raised manually and not from msgprint or throw message = "{0}".format(e.__class__.__name__) if e.args: - message += " : {0}".format(e.args[0]) + message += " : {0}".format(e.args[0]) message_dict = {"docname": docname, "message": message} failed_transactions[docname].append(message_dict) diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py index ab6ffd4985..45e008fa04 100644 --- a/frappe/modules/export_file.py +++ b/frappe/modules/export_file.py @@ -47,7 +47,7 @@ def strip_default_fields(doc, doc_export): for df in doc.meta.get_table_fields(): for d in doc_export.get(df.fieldname): - for fieldname in frappe.model.default_fields: + for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): if fieldname in d: del d[fieldname] diff --git a/frappe/patches.txt b/frappe/patches.txt index 9880596e27..db9610a767 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -119,7 +119,6 @@ execute:frappe.delete_doc_if_exists('DocType', 'GSuite Settings') execute:frappe.delete_doc_if_exists('DocType', 'GSuite Templates') execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Account') execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings') -frappe.patches.v12_0.remove_parent_and_parenttype_from_print_formats frappe.patches.v12_0.remove_example_email_thread_notify execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders() frappe.patches.v12_0.set_correct_url_in_files @@ -185,7 +184,6 @@ frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty frappe.patches.v13_0.set_first_day_of_the_week -frappe.patches.v14_0.rename_cancelled_documents frappe.patches.v14_0.update_workspace2 # 20.09.2021 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021 frappe.patches.v14_0.transform_todo_schema diff --git a/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py b/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py deleted file mode 100644 index 1a3c56da59..0000000000 --- a/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py +++ /dev/null @@ -1,14 +0,0 @@ -import frappe - -def execute(): - frappe.db.sql(""" - UPDATE - `tabPrint Format` - SET - `tabPrint Format`.`parent`='', - `tabPrint Format`.`parenttype`='', - `tabPrint Format`.parentfield='' - WHERE - `tabPrint Format`.parent != '' - OR `tabPrint Format`.parenttype != '' - """) \ No newline at end of file diff --git a/frappe/patches/v13_0/remove_chat.py b/frappe/patches/v13_0/remove_chat.py index d884419cac..1804c7693f 100644 --- a/frappe/patches/v13_0/remove_chat.py +++ b/frappe/patches/v13_0/remove_chat.py @@ -3,6 +3,7 @@ import click def execute(): frappe.delete_doc_if_exists("DocType", "Chat Message") + frappe.delete_doc_if_exists("DocType", "Chat Message Attachment") frappe.delete_doc_if_exists("DocType", "Chat Profile") frappe.delete_doc_if_exists("DocType", "Chat Token") frappe.delete_doc_if_exists("DocType", "Chat Room User") diff --git a/frappe/patches/v14_0/rename_cancelled_documents.py b/frappe/patches/v14_0/rename_cancelled_documents.py deleted file mode 100644 index 4b565d4f76..0000000000 --- a/frappe/patches/v14_0/rename_cancelled_documents.py +++ /dev/null @@ -1,213 +0,0 @@ -import functools -import traceback - -import frappe - -def execute(): - """Rename cancelled documents by adding a postfix. - """ - rename_cancelled_docs() - -def get_submittable_doctypes(): - """Returns list of submittable doctypes in the system. - """ - return frappe.db.get_all('DocType', filters={'is_submittable': 1}, pluck='name') - -def get_cancelled_doc_names(doctype): - """Return names of cancelled document names those are in old format. - """ - docs = frappe.db.get_all(doctype, filters={'docstatus': 2}, pluck='name') - return [each for each in docs if not (each.endswith('-CANC') or ('-CANC-' in each))] - -@functools.lru_cache() -def get_linked_doctypes(): - """Returns list of doctypes those are linked with given doctype using 'Link' fieldtype. - """ - filters=[['fieldtype','=', 'Link']] - links = frappe.get_all("DocField", - fields=["parent", "fieldname", "options as linked_to"], - filters=filters, - as_list=1) - - links+= frappe.get_all("Custom Field", - fields=["dt as parent", "fieldname", "options as linked_to"], - filters=filters, - as_list=1) - - links_by_doctype = {} - for doctype, fieldname, linked_to in links: - links_by_doctype.setdefault(linked_to, []).append((doctype, fieldname)) - return links_by_doctype - -@functools.lru_cache() -def get_single_doctypes(): - return frappe.get_all("DocType", filters={'issingle': 1}, pluck='name') - -@functools.lru_cache() -def get_dynamic_linked_doctypes(): - filters=[['fieldtype','=', 'Dynamic Link']] - - # find dynamic links of parents - links = frappe.get_all("DocField", - fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], - filters=filters, - as_list=1) - links+= frappe.get_all("Custom Field", - fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], - filters=filters, - as_list=1) - return links - -@functools.lru_cache() -def get_child_tables(): - """ - """ - filters =[['fieldtype', 'in', ('Table', 'Table MultiSelect')]] - links = frappe.get_all("DocField", - fields=["parent as doctype", "options as child_table"], - filters=filters, - as_list=1) - - links+= frappe.get_all("Custom Field", - fields=["dt as doctype", "options as child_table"], - filters=filters, - as_list=1) - - map = {} - for doctype, child_table in links: - map.setdefault(doctype, []).append(child_table) - return map - -def update_cancelled_document_names(doctype, cancelled_doc_names): - return frappe.db.sql(""" - update - `tab{doctype}` - set - name=CONCAT(name, '-CANC') - where - docstatus=2 - and - name in %(cancelled_doc_names)s; - """.format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names}) - -def update_amended_field(doctype, cancelled_doc_names): - return frappe.db.sql(""" - update - `tab{doctype}` - set - amended_from=CONCAT(amended_from, '-CANC') - where - amended_from in %(cancelled_doc_names)s; - """.format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names}) - -def update_attachments(doctype, cancelled_doc_names): - frappe.db.sql(""" - update - `tabFile` - set - attached_to_name=CONCAT(attached_to_name, '-CANC') - where - attached_to_doctype=%(dt)s and attached_to_name in %(cancelled_doc_names)s - """, {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - -def update_versions(doctype, cancelled_doc_names): - frappe.db.sql(""" - UPDATE - `tabVersion` - SET - docname=CONCAT(docname, '-CANC') - WHERE - ref_doctype=%(dt)s AND docname in %(cancelled_doc_names)s - """, {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - -def update_linked_doctypes(doctype, cancelled_doc_names): - single_doctypes = get_single_doctypes() - - for linked_dt, field in get_linked_doctypes().get(doctype, []): - if linked_dt not in single_doctypes: - frappe.db.sql(""" - update - `tab{linked_dt}` - set - `{column}`=CONCAT(`{column}`, '-CANC') - where - `{column}` in %(cancelled_doc_names)s; - """.format(linked_dt=linked_dt, column=field), - {'cancelled_doc_names': cancelled_doc_names}) - else: - doc = frappe.get_single(linked_dt) - if getattr(doc, field) in cancelled_doc_names: - setattr(doc, field, getattr(doc, field)+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def update_dynamic_linked_doctypes(doctype, cancelled_doc_names): - single_doctypes = get_single_doctypes() - - for linked_dt, fieldname, doctype_fieldname in get_dynamic_linked_doctypes(): - if linked_dt not in single_doctypes: - frappe.db.sql(""" - update - `tab{linked_dt}` - set - `{column}`=CONCAT(`{column}`, '-CANC') - where - `{column}` in %(cancelled_doc_names)s and {doctype_fieldname}=%(dt)s; - """.format(linked_dt=linked_dt, column=fieldname, doctype_fieldname=doctype_fieldname), - {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - else: - doc = frappe.get_single(linked_dt) - if getattr(doc, doctype_fieldname) == doctype and getattr(doc, fieldname) in cancelled_doc_names: - setattr(doc, fieldname, getattr(doc, fieldname)+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def update_child_tables(doctype, cancelled_doc_names): - child_tables = get_child_tables().get(doctype, []) - single_doctypes = get_single_doctypes() - - for table in child_tables: - if table not in single_doctypes: - frappe.db.sql(""" - update - `tab{table}` - set - parent=CONCAT(parent, '-CANC') - where - parenttype=%(dt)s and parent in %(cancelled_doc_names)s; - """.format(table=table), {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - else: - doc = frappe.get_single(table) - if getattr(doc, 'parenttype')==doctype and getattr(doc, 'parent') in cancelled_doc_names: - setattr(doc, 'parent', getattr(doc, 'parent')+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def rename_cancelled_docs(): - submittable_doctypes = get_submittable_doctypes() - - for dt in submittable_doctypes: - for retry in range(2): - try: - cancelled_doc_names = tuple(get_cancelled_doc_names(dt)) - if not cancelled_doc_names: - break - update_cancelled_document_names(dt, cancelled_doc_names) - update_amended_field(dt, cancelled_doc_names) - update_child_tables(dt, cancelled_doc_names) - update_linked_doctypes(dt, cancelled_doc_names) - update_dynamic_linked_doctypes(dt, cancelled_doc_names) - update_attachments(dt, cancelled_doc_names) - update_versions(dt, cancelled_doc_names) - print(f"Renaming cancelled records of {dt} doctype") - frappe.db.commit() - break - except Exception: - if retry == 1: - print(f"Failed to rename the cancelled records of {dt} doctype, moving on!") - traceback.print_exc() - frappe.db.rollback() - diff --git a/frappe/public/js/frappe-web.bundle.js b/frappe/public/js/frappe-web.bundle.js index c962457964..b8d4006090 100644 --- a/frappe/public/js/frappe-web.bundle.js +++ b/frappe/public/js/frappe-web.bundle.js @@ -2,6 +2,7 @@ import "./jquery-bootstrap"; import "./frappe/class.js"; import "./frappe/polyfill.js"; import "./lib/md5.min.js"; +import "./lib/moment.js"; import "./frappe/provide.js"; import "./frappe/format.js"; import "./frappe/utils/number_format.js"; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 2264042539..b153718c70 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -331,7 +331,7 @@ frappe.data_import.ImportPreview = class ImportPreview { is_row_imported(row) { let serial_no = row[0].content; return this.import_log.find(log => { - return log.success && log.row_indexes.includes(serial_no); + return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no); }); } }; diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index 167b4955fa..1b30726a7a 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -534,22 +534,21 @@ export default { }); }, show_google_drive_picker() { - let dialog = cur_dialog; - dialog.hide(); + this.close_dialog = true; let google_drive = new GoogleDrivePicker({ - pickerCallback: data => this.google_drive_callback(data, dialog), + pickerCallback: data => this.google_drive_callback(data), ...this.google_drive_settings }); google_drive.loadPicker(); }, - google_drive_callback(data, dialog) { + google_drive_callback(data) { if (data.action == google.picker.Action.PICKED) { this.upload_file({ file_url: data.docs[0].url, file_name: data.docs[0].name }); } else if (data.action == google.picker.Action.CANCEL) { - dialog.show(); + cur_frm.attachments.new_attachment() } }, url_to_file(url, filename, mime_type) { diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index ed355cf8b4..9f02485a9e 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -374,10 +374,22 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat } set_custom_query(args) { - var set_nulls = function(obj) { - $.each(obj, function(key, value) { - if(value!==undefined) { - obj[key] = value; + const is_valid_value = (value, key) => { + if (value) return true; + // check if empty value is valid + if (this.frm) { + let field = frappe.meta.get_docfield(this.frm.doctype, key); + // empty value link fields is invalid + return !field || !["Link", "Dynamic Link"].includes(field.fieldtype); + } else { + return value !== undefined; + } + } + + const set_nulls = (obj) => { + $.each(obj, (key, value) => { + if (!is_valid_value(value, key)) { + delete obj[key]; } }); return obj; @@ -458,7 +470,6 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat validate_link_and_fetch(df, options, docname, value) { if (!options) return; - let field_value = ""; const fetch_map = this.fetch_map; const columns_to_fetch = Object.values(fetch_map); @@ -467,16 +478,10 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat return value; } - return frappe.xcall("frappe.client.validate_link", { - doctype: options, - docname: value, - fields: columns_to_fetch, - }).then((response) => { - if (!docname || !columns_to_fetch.length) return response.name; - + function update_dependant_fields(response) { + let field_value = ""; for (const [target_field, source_field] of Object.entries(fetch_map)) { if (value) field_value = response[source_field]; - frappe.model.set_value( df.parent, docname, @@ -485,9 +490,23 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat df.fieldtype, ); } + } - return response.name; - }); + // to avoid unnecessary request + if (value) { + return frappe.xcall("frappe.client.validate_link", { + doctype: options, + docname: value, + fields: columns_to_fetch, + }).then((response) => { + if (!docname || !columns_to_fetch.length) return response.name; + update_dependant_fields(response); + return response.name; + }); + } else { + update_dependant_fields({}); + return value; + } } get fetch_map() { diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 5c0b6b1399..c0c7ce8b4e 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -860,36 +860,32 @@ frappe.ui.form.Form = class FrappeForm { } _cancel(btn, callback, on_error, skip_confirm) { + const me = this; const cancel_doc = () => { frappe.validated = true; - this.script_manager.trigger("before_cancel").then(() => { + me.script_manager.trigger("before_cancel").then(() => { if (!frappe.validated) { - return this.handle_save_fail(btn, on_error); + return me.handle_save_fail(btn, on_error); } - const original_name = this.docname; - const after_cancel = (r) => { + var after_cancel = function(r) { if (r.exc) { - this.handle_save_fail(btn, on_error); + me.handle_save_fail(btn, on_error); } else { frappe.utils.play_sound("cancel"); + me.refresh(); callback && callback(); - this.script_manager.trigger("after_cancel"); - frappe.run_serially([ - () => this.rename_notify(this.doctype, original_name, r.docs[0].name), - () => frappe.router.clear_re_route(this.doctype, original_name), - () => this.refresh(), - ]); + me.script_manager.trigger("after_cancel"); } }; - frappe.ui.form.save(this, "cancel", after_cancel, btn); + frappe.ui.form.save(me, "cancel", after_cancel, btn); }); } if (skip_confirm) { cancel_doc(); } else { - frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error)); + frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error)); } }; @@ -911,7 +907,7 @@ frappe.ui.form.Form = class FrappeForm { 'docname': this.doc.name }).then(is_amended => { if (is_amended) { - frappe.throw(__('This document is already amended, you cannot amend it again')); + frappe.throw(__('This document is already amended, you cannot ammend it again')); } this.validate_form_action("Amend"); var me = this; diff --git a/frappe/public/js/frappe/form/multi_select_dialog.js b/frappe/public/js/frappe/form/multi_select_dialog.js index bc0286e62d..bc5f7a9b52 100644 --- a/frappe/public/js/frappe/form/multi_select_dialog.js +++ b/frappe/public/js/frappe/form/multi_select_dialog.js @@ -150,8 +150,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { }); } + is_child_selection_enabled() { + return this.dialog.fields_dict['allow_child_item_selection'].get_value(); + } + toggle_child_selection() { - if (this.dialog.fields_dict['allow_child_item_selection'].get_value()) { + if (this.is_child_selection_enabled()) { this.show_child_results(); } else { this.child_results = []; @@ -289,7 +293,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { parent: this.dialog.get_field('filter_area').$wrapper, doctype: this.doctype, on_change: () => { - this.get_results(); + if (this.is_child_selection_enabled()) { + this.show_child_results(); + } else { + this.get_results(); + } } }); // 'Apply Filter' breaks since the filers are not in a popover @@ -325,7 +333,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { this.$parent.find('.input-with-feedback').on('change', () => { frappe.flags.auto_scroll = false; - this.get_results(); + if (this.is_child_selection_enabled()) { + this.show_child_results(); + } else { + this.get_results(); + } }); this.$parent.find('[data-fieldtype="Data"]').on('input', () => { @@ -333,8 +345,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { clearTimeout($this.data('timeout')); $this.data('timeout', setTimeout(function () { frappe.flags.auto_scroll = false; - me.empty_list(); - me.get_results(); + if (me.is_child_selection_enabled()) { + me.show_child_results(); + } else { + me.empty_list(); + me.get_results(); + } }, 300)); }); } diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index 041905408a..89e029ffb1 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -10,8 +10,7 @@ $.extend(frappe.model, { layout_fields: ['Section Break', 'Column Break', 'Tab Break', 'Fold'], std_fields_list: ['name', 'owner', 'creation', 'modified', 'modified_by', - '_user_tags', '_comments', '_assign', '_liked_by', 'docstatus', - 'parent', 'parenttype', 'parentfield', 'idx'], + '_user_tags', '_comments', '_assign', '_liked_by', 'docstatus', 'idx'], core_doctypes_list: ['DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role', 'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form', diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 7bea7f0584..1b038b6265 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -250,12 +250,6 @@ frappe.router = { } }, - clear_re_route(doctype, docname) { - delete frappe.re_route[ - `${encodeURIComponent(frappe.router.slug(doctype))}/${encodeURIComponent(docname)}` - ]; - }, - set_title(sub_path) { if (frappe.route_titles[sub_path]) { frappe.utils.set_title(frappe.route_titles[sub_path]); diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index ae0a2edcda..4026f9b47b 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -70,6 +70,9 @@ frappe.breadcrumbs = { this.set_form_breadcrumb(breadcrumbs, view); } else if (breadcrumbs.doctype && view === 'list') { this.set_list_breadcrumb(breadcrumbs); + } else if (breadcrumbs.doctype && view == 'dashboard-view') { + this.set_list_breadcrumb(breadcrumbs); + this.set_dashboard_breadcrumb(breadcrumbs); } } @@ -164,6 +167,14 @@ frappe.breadcrumbs = { }, + set_dashboard_breadcrumb(breadcrumbs) { + const doctype = breadcrumbs.doctype; + const docname = frappe.get_route()[1]; + let dashboard_route = `/app/${frappe.router.slug(doctype)}/${docname}`; + $(`
  • ${__(docname)}
    • `) + .appendTo(this.$breadcrumbs); + }, + setup_modules() { if (!frappe.visible_modules) { frappe.visible_modules = $.map(frappe.boot.allowed_workspaces, (m) => { diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index f5d9f3e110..23e415ed3e 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -866,7 +866,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { } doctype_fields = [{ - label: __('ID'), + label: __('ID', null, 'Label of name column in report'), fieldname: 'name', fieldtype: 'Data', reqd: 1 diff --git a/frappe/public/js/integrations/google_drive_picker.js b/frappe/public/js/integrations/google_drive_picker.js index 9d7971e75c..1e4f1dca7c 100644 --- a/frappe/public/js/integrations/google_drive_picker.js +++ b/frappe/public/js/integrations/google_drive_picker.js @@ -44,9 +44,16 @@ export default class GoogleDrivePicker { } handleAuthResult(authResult) { + let error_map = { + "popup_closed_by_user": __("Google Authentication was closed abruptly by the user") + }; + if (authResult && !authResult.error) { frappe.boot.user.google_drive_token = authResult.access_token; this.createPicker(); + } else { + let error = error_map[authResult.error] || __("Google Authentication Error"); + frappe.throw(error); } } @@ -58,20 +65,34 @@ export default class GoogleDrivePicker { createPicker() { // Create and render a Picker object for searching images. if (this.pickerApiLoaded && frappe.boot.user.google_drive_token) { - var view = new google.picker.DocsView(google.picker.ViewId.DOCS) + this.view = new google.picker.DocsView(google.picker.ViewId.DOCS) .setParent('root') // show the root folder by default .setIncludeFolders(true); // also show folders, not just files - var picker = new google.picker.PickerBuilder() + this.picker = new google.picker.PickerBuilder() .setAppId(this.appId) .setDeveloperKey(this.developerKey) .setOAuthToken(frappe.boot.user.google_drive_token) - .addView(view) + .addView(this.view) .setLocale(frappe.boot.lang) .setCallback(this.pickerCallback) .build(); - picker.setVisible(true); + this.picker.setVisible(true); + this.setupHide(); + } + } + + setupHide() { + let bg = $(".picker-dialog-bg"); + + for (let el of bg) { + el.onclick = () => { + this.picker.setVisible(false); + this.picker.Ob({ + action: google.picker.Action.CANCEL + }); + }; } } } diff --git a/frappe/public/js/lib/moment.js b/frappe/public/js/lib/moment.js new file mode 100644 index 0000000000..7a817a36cd --- /dev/null +++ b/frappe/public/js/lib/moment.js @@ -0,0 +1,5 @@ +// This file is used to make sure that `moment` is bound to the window +// before the bundle finishes loading, due to imports (datetime.js) in the bundle +// that depend on `moment`. +import momentTimezone from "moment-timezone/builds/moment-timezone-with-data.js"; +window.moment = momentTimezone; diff --git a/frappe/public/js/libs.bundle.js b/frappe/public/js/libs.bundle.js index 876d76875b..b71cc592a0 100644 --- a/frappe/public/js/libs.bundle.js +++ b/frappe/public/js/libs.bundle.js @@ -1,15 +1,12 @@ import "./jquery-bootstrap"; import Vue from "vue/dist/vue.esm.js"; -import moment from "moment/min/moment-with-locales.js"; -import momentTimezone from "moment-timezone/builds/moment-timezone-with-data.js"; +import "./lib/moment"; import io from "socket.io-client/dist/socket.io.slim.js"; import Sortable from "./lib/Sortable.min.js"; // TODO: esbuild // Don't think jquery.hotkeys is being used anywhere. Will remove this after being sure. // import "./lib/jquery/jquery.hotkeys.js"; -window.moment = moment; -window.moment = momentTimezone; window.Vue = Vue; window.Sortable = Sortable; window.io = io; diff --git a/frappe/public/js/print_format_builder/print_format_builder.bundle.js b/frappe/public/js/print_format_builder/print_format_builder.bundle.js index b2d3372daf..c8c03d209a 100644 --- a/frappe/public/js/print_format_builder/print_format_builder.bundle.js +++ b/frappe/public/js/print_format_builder/print_format_builder.bundle.js @@ -21,7 +21,7 @@ class PrintFormatBuilder { this.$component.toggle_preview(); } ); - this.page.add_button(__("Reset Changes"), () => + let $reset_changes_btn = this.page.add_button(__("Reset Changes"), () => this.$component.$store.reset_changes() ); this.page.add_menu_item(__("Edit Print Format"), () => { @@ -46,9 +46,11 @@ class PrintFormatBuilder { if (value) { this.page.set_indicator("Not Saved", "orange"); $toggle_preview_btn.hide(); + $reset_changes_btn.show(); } else { this.page.clear_indicator(); $toggle_preview_btn.show(); + $reset_changes_btn.hide(); } }); this.$component.$watch("show_preview", value => { diff --git a/frappe/public/js/web_form.bundle.js b/frappe/public/js/web_form.bundle.js index 01969a489c..ffb7b824bd 100644 --- a/frappe/public/js/web_form.bundle.js +++ b/frappe/public/js/web_form.bundle.js @@ -1,2 +1,3 @@ +import "./lib/moment.js"; import "./frappe/utils/datetime.js"; import "./frappe/web_form/webform_script.js"; diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index f4794362d3..b8b7f869fa 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -249,6 +249,7 @@ --checkbox-right-margin: var(--margin-xs); --checkbox-size: 14px; --checkbox-focus-shadow: 0 0 0 2px var(--gray-300); + --checkbox-gradient: linear-gradient(180deg, #4AC3F8 -124.51%, var(--primary) 100%); --right-arrow-svg: url("data: image/svg+xml;utf8, "); --left-arrow-svg: url("data: image/svg+xml;utf8, "); diff --git a/frappe/public/scss/common/global.scss b/frappe/public/scss/common/global.scss index 0324b75bfb..8a849ab51a 100644 --- a/frappe/public/scss/common/global.scss +++ b/frappe/public/scss/common/global.scss @@ -54,7 +54,7 @@ input[type="radio"] { } &:checked::before { - background-color: var(--blue-500); + background-color: var(--primary); border-radius: 16px; box-shadow: inset 0 0 0 2px white; } @@ -85,8 +85,8 @@ input[type="checkbox"] { } &:checked { - background-color: var(--blue-500); - background-image: $check-icon, linear-gradient(180deg, #4AC3F8 -124.51%, #2490EF 100%); + background-color: var(--primary); + background-image: $check-icon, var(--checkbox-gradient); background-size: 57%, 100%; box-shadow: none; border: none; diff --git a/frappe/public/scss/desk/css_variables.scss b/frappe/public/scss/desk/css_variables.scss index 0912cb278b..a06ba3e9b0 100644 --- a/frappe/public/scss/desk/css_variables.scss +++ b/frappe/public/scss/desk/css_variables.scss @@ -54,4 +54,6 @@ $input-height: 28px !default; // skeleton --skeleton-bg: var(--gray-100); + // progress bar + --progress-bar-bg: var(--primary); } diff --git a/frappe/public/scss/desk/variables.scss b/frappe/public/scss/desk/variables.scss index 2855277ccd..abc63cd637 100644 --- a/frappe/public/scss/desk/variables.scss +++ b/frappe/public/scss/desk/variables.scss @@ -118,6 +118,9 @@ $custom-control-label-color: var(--text-color); $custom-switch-indicator-size: 8px; $custom-control-indicator-border-width: 2px; +// progress bar +$progress-bar-bg: var(--progress-bar-bg); + $navbar-nav-link-padding-x: 1rem !default; $navbar-padding-y: 1rem !default; $card-border-radius: 0.75rem !default; diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index bf7be84c51..5b58e70c4e 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1,8 +1,21 @@ -from frappe.query_builder.terms import ParameterizedValueWrapper, ParameterizedFunction -import pypika +import pypika.terms +from pypika import * +from pypika import Field +from pypika.utils import ignore_copy + +from frappe.query_builder.terms import ParameterizedFunction, ParameterizedValueWrapper +from frappe.query_builder.utils import ( + Column, + DocType, + get_query_builder, + patch_query_aggregation, + patch_query_execute, +) pypika.terms.ValueWrapper = ParameterizedValueWrapper pypika.terms.Function = ParameterizedFunction -from pypika import * -from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute, patch_query_aggregation +# * Overrides the field() method and replaces it with the a `PseudoColumn` 'field' for consistency +pypika.queries.Selectable.__getattr__ = ignore_copy(lambda table, x: Field(x, table=table)) +pypika.queries.Selectable.__getitem__ = ignore_copy(lambda table, x: Field(x, table=table)) +pypika.queries.Selectable.field = pypika.terms.PseudoColumn("field") diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index a65d50fdeb..d2fdeab324 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -1,8 +1,12 @@ from pypika import MySQLQuery, Order, PostgreSQLQuery, terms -from pypika.queries import Schema, Table -from frappe.utils import get_table_name +from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder +from pypika.queries import QueryBuilder, Schema, Table from pypika.terms import Function +from frappe.query_builder.terms import ParameterizedValueWrapper +from frappe.utils import get_table_name + + class Base: terms = terms desc = Order.desc @@ -19,13 +23,13 @@ class Base: return Table(table_name, *args, **kwargs) @classmethod - def into(cls, table, *args, **kwargs): + def into(cls, table, *args, **kwargs) -> QueryBuilder: if isinstance(table, str): table = cls.DocType(table) return super().into(table, *args, **kwargs) @classmethod - def update(cls, table, *args, **kwargs): + def update(cls, table, *args, **kwargs) -> QueryBuilder: if isinstance(table, str): table = cls.DocType(table) return super().update(table, *args, **kwargs) @@ -34,6 +38,10 @@ class Base: class MariaDB(Base, MySQLQuery): Field = terms.Field + @classmethod + def _builder(cls, *args, **kwargs) -> "MySQLQueryBuilder": + return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) + @classmethod def from_(cls, table, *args, **kwargs): if isinstance(table, str): @@ -53,6 +61,10 @@ class Postgres(Base, PostgreSQLQuery): # they are two different objects. The quick fix used here is to replace the # Field names in the "Field" function. + @classmethod + def _builder(cls, *args, **kwargs) -> "PostgreSQLQueryBuilder": + return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) + @classmethod def Field(cls, field_name, *args, **kwargs): if field_name in cls.field_translation: diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index 2032cd8497..205f1f9dcd 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -1,33 +1,77 @@ +from datetime import timedelta from typing import Any, Dict, Optional +from frappe.utils.data import format_timedelta from pypika.terms import Function, ValueWrapper from pypika.utils import format_alias_sql -class NamedParameterWrapper(): - def __init__(self, parameters: Dict[str, Any]): - self.parameters = parameters +class NamedParameterWrapper: + """Utility class to hold parameter values and keys""" - def update_parameters(self, param_key: Any, param_value: Any, **kwargs): + def __init__(self) -> None: + self.parameters = {} + + def get_sql(self, param_value: Any, **kwargs) -> str: + """returns SQL for a parameter, while adding the real value in a dict + + Args: + param_value (Any): Value of the parameter + + Returns: + str: parameter used in the SQL query + """ + param_key = f"%(param{len(self.parameters) + 1})s" self.parameters[param_key[2:-2]] = param_value + return param_key - def get_sql(self, **kwargs): - return f'%(param{len(self.parameters) + 1})s' + def get_parameters(self) -> Dict[str, Any]: + """get dict with parameters and values + + Returns: + Dict[str, Any]: parameter dict + """ + return self.parameters class ParameterizedValueWrapper(ValueWrapper): - def get_sql(self, quote_char: Optional[str] = None, secondary_quote_char: str = "'", param_wrapper= None, **kwargs: Any) -> str: - if param_wrapper is None: - sql = self.get_value_sql(quote_char=quote_char, secondary_quote_char=secondary_quote_char, **kwargs) - return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs) + """ + Class to monkey patch ValueWrapper + + Adds functionality to parameterize queries when a `param wrapper` is passed in get_sql() + """ + + def get_sql( + self, + quote_char: Optional[str] = None, + secondary_quote_char: str = "'", + param_wrapper: Optional[NamedParameterWrapper] = None, + **kwargs: Any, + ) -> str: + if param_wrapper and isinstance(self.value, str): + # add quotes if it's a string value + value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) + sql = param_wrapper.get_sql(param_value=value_sql, **kwargs) else: - value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) if not isinstance(self.value,int) else self.value - param_sql = param_wrapper.get_sql(**kwargs) - param_wrapper.update_parameters(param_key=param_sql, param_value=value_sql, **kwargs) - return format_alias_sql(param_sql, self.alias, quote_char=quote_char, **kwargs) + # * BUG: pypika doesen't parse timedeltas + if isinstance(self.value, timedelta): + self.value = format_timedelta(self.value) + sql = self.get_value_sql( + quote_char=quote_char, + secondary_quote_char=secondary_quote_char, + param_wrapper=param_wrapper, + **kwargs, + ) + return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs) class ParameterizedFunction(Function): + """ + Class to monkey patch pypika.terms.Functions + + Only to pass `param_wrapper` in `get_function_sql`. + """ + def get_sql(self, **kwargs: Any) -> str: with_alias = kwargs.pop("with_alias", False) with_namespace = kwargs.pop("with_namespace", False) @@ -35,15 +79,24 @@ class ParameterizedFunction(Function): dialect = kwargs.pop("dialect", None) param_wrapper = kwargs.pop("param_wrapper", None) - function_sql = self.get_function_sql(with_namespace=with_namespace, quote_char=quote_char, param_wrapper=param_wrapper, dialect=dialect) + function_sql = self.get_function_sql( + with_namespace=with_namespace, + quote_char=quote_char, + param_wrapper=param_wrapper, + dialect=dialect, + ) if self.schema is not None: function_sql = "{schema}.{function}".format( - schema=self.schema.get_sql(quote_char=quote_char, dialect=dialect, **kwargs), + schema=self.schema.get_sql( + quote_char=quote_char, dialect=dialect, **kwargs + ), function=function_sql, ) if with_alias: - return format_alias_sql(function_sql, self.alias, quote_char=quote_char, **kwargs) + return format_alias_sql( + function_sql, self.alias, quote_char=quote_char, **kwargs + ) return function_sql diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 2767e90242..1ddf4fc034 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -1,17 +1,17 @@ from enum import Enum -from typing import Any, Callable, Dict, Union, get_type_hints from importlib import import_module +from typing import Any, Callable, Dict, Union, get_type_hints from pypika import Query from pypika.queries import Column - -import frappe - -from .builder import MariaDB, Postgres from pypika.terms import PseudoColumn +import frappe from frappe.query_builder.terms import NamedParameterWrapper +from .builder import MariaDB, Postgres + + class db_type_is(Enum): MARIADB = "mariadb" POSTGRES = "postgres" @@ -59,11 +59,29 @@ def patch_query_execute(): return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep def prepare_query(query): - params = {} - query = query.get_sql(param_wrapper = NamedParameterWrapper(params)) + import inspect + + param_collector = NamedParameterWrapper() + query = query.get_sql(param_wrapper=param_collector) if frappe.flags.in_safe_exec and not query.lower().strip().startswith("select"): - raise frappe.PermissionError('Only SELECT SQL allowed in scripting') - return query, params + callstack = inspect.stack() + if len(callstack) >= 3 and ".py" in callstack[2].filename: + # ignore any query builder methods called from python files + # assumption is that those functions are whitelisted already. + + # since query objects are patched everywhere any query.run() + # will have callstack like this: + # frame0: this function prepare_query() + # frame1: execute_query() + # frame2: frame that called `query.run()` + # + # if frame2 is server script it wont have a filename and hence + # it shouldn't be allowed. + # ps. stack() returns `""` as filename. + pass + else: + raise frappe.PermissionError('Only SELECT SQL allowed in scripting') + return query, param_collector.get_parameters() query_class = get_attr(str(frappe.qb).split("'")[1]) builder_class = get_type_hints(query_class._builder).get('return') @@ -78,7 +96,7 @@ def patch_query_execute(): def patch_query_aggregation(): """Patch aggregation functions to frappe.qb """ - from frappe.query_builder.functions import _max, _min, _avg, _sum + from frappe.query_builder.functions import _avg, _max, _min, _sum frappe.qb.max = _max frappe.qb.min = _min diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py index 1d4f3fef32..79ccd3c6d5 100644 --- a/frappe/search/full_text_search.py +++ b/frappe/search/full_text_search.py @@ -66,7 +66,7 @@ class FullTextSearch: ix = self.get_index() with ix.searcher(): - writer = ix.writer() + writer = AsyncWriter(ix) writer.delete_by_term(self.id, doc_name) writer.commit(optimize=True) @@ -98,7 +98,7 @@ class FullTextSearch: def build_index(self): """Build index for all parsed documents""" ix = self.create_index() - writer = ix.writer() + writer = AsyncWriter(ix) for i, document in enumerate(self.documents): if document: diff --git a/frappe/social/doctype/energy_point_rule/energy_point_rule.py b/frappe/social/doctype/energy_point_rule/energy_point_rule.py index ab860eb1aa..55bf55a3b0 100644 --- a/frappe/social/doctype/energy_point_rule/energy_point_rule.py +++ b/frappe/social/doctype/energy_point_rule/energy_point_rule.py @@ -59,9 +59,9 @@ class EnergyPointRule(Document): # indicates that this was a new doc return doc.get_doc_before_save() is None if self.for_doc_event == 'Submit': - return doc.docstatus == 1 + return doc.docstatus.is_submitted() if self.for_doc_event == 'Cancel': - return doc.docstatus == 2 + return doc.docstatus.is_cancelled() if self.for_doc_event == 'Value Change': field_to_check = self.field_to_check if not field_to_check: return False @@ -96,7 +96,7 @@ def process_energy_points(doc, state): old_doc = doc.get_doc_before_save() # check if doc has been cancelled - if old_doc and old_doc.docstatus == 1 and doc.docstatus == 2: + if old_doc and old_doc.docstatus.is_submitted() and doc.docstatus.is_cancelled(): return revert_points_for_cancelled_doc(doc) for d in frappe.cache_manager.get_doctype_map('Energy Point Rule', doc.doctype, diff --git a/frappe/templates/base.html b/frappe/templates/base.html index bc1f802cf7..8d892b5de6 100644 --- a/frappe/templates/base.html +++ b/frappe/templates/base.html @@ -105,8 +105,6 @@ // for backward compatibility of some libs frappe.sys_defaults = frappe.boot.sysdefaults; - - {{ include_script('frappe-web.bundle.js') }} {% endblock %} diff --git a/frappe/templates/print_format/print_format.css b/frappe/templates/print_format/print_format.css index 480cd19439..baaf5b087d 100644 --- a/frappe/templates/print_format/print_format.css +++ b/frappe/templates/print_format/print_format.css @@ -57,6 +57,10 @@ body { padding-left: {{ print_format.margin_left | int }}mm; padding-bottom: {{ print_format.margin_bottom | int }}mm; } + + .child-table { + overflow-x: auto; + } } .section:not(:first-child) { diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 1839f15ae8..05f1ce1cd7 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -30,7 +30,7 @@ def xmlrunner_wrapper(output): def main(app=None, module=None, doctype=None, verbose=False, tests=(), force=False, profile=False, junit_xml_output=None, ui_tests=False, - doctype_list_path=None, skip_test_records=False, failfast=False): + doctype_list_path=None, skip_test_records=False, failfast=False, case=None): global unittest_runner if doctype_list_path: @@ -76,7 +76,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), if doctype: ret = run_tests_for_doctype(doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output) elif module: - ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output) + ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case) else: ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast, junit_xml_output=junit_xml_output) @@ -182,16 +182,16 @@ def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profil return _run_unittest(modules, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output) -def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False): +def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None): module = importlib.import_module(module) if hasattr(module, "test_dependencies"): for doctype in module.test_dependencies: make_test_records(doctype, verbose=verbose) frappe.db.commit() - return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output) + return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case) -def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False): +def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None): frappe.db.begin() test_suite = unittest.TestSuite() @@ -200,7 +200,10 @@ def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=Fals modules = [modules] for module in modules: - module_test_cases = unittest.TestLoader().loadTestsFromModule(module) + if case: + module_test_cases = unittest.TestLoader().loadTestsFromTestCase(getattr(module, case)) + else: + module_test_cases = unittest.TestLoader().loadTestsFromModule(module) if tests: for each in module_test_cases: for test_case in each.__dict__["_tests"]: @@ -337,7 +340,7 @@ def make_test_records_for_doctype(doctype, verbose=0, force=False): elif hasattr(test_module, "test_records"): if doctype in frappe.local.test_objects: frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force) - else: + else: frappe.local.test_objects[doctype] = make_test_objects(doctype, test_module.test_records, verbose, force) else: diff --git a/frappe/tests/test_base_document.py b/frappe/tests/test_base_document.py new file mode 100644 index 0000000000..7e165e9045 --- /dev/null +++ b/frappe/tests/test_base_document.py @@ -0,0 +1,18 @@ +import unittest + +from frappe.model.base_document import BaseDocument + + +class TestBaseDocument(unittest.TestCase): + def test_docstatus(self): + doc = BaseDocument({"docstatus": 0}) + self.assertTrue(doc.docstatus.is_draft()) + self.assertEquals(doc.docstatus, 0) + + doc.docstatus = 1 + self.assertTrue(doc.docstatus.is_submitted()) + self.assertEquals(doc.docstatus, 1) + + doc.docstatus = 2 + self.assertTrue(doc.docstatus.is_cancelled()) + self.assertEquals(doc.docstatus, 2) diff --git a/frappe/tests/test_child_table.py b/frappe/tests/test_child_table.py new file mode 100644 index 0000000000..8cdfd08599 --- /dev/null +++ b/frappe/tests/test_child_table.py @@ -0,0 +1,66 @@ +import frappe +from frappe.model import child_table_fields + +import unittest +from typing import Callable + + +class TestChildTable(unittest.TestCase): + def tearDown(self) -> None: + try: + frappe.delete_doc("DocType", self.doctype_name, force=1) + except Exception: + pass + + def test_child_table_doctype_creation_and_transitioning(self) -> None: + ''' + This method tests the creation of child table doctype + as well as it's transitioning from child table to normal and normal to child table doctype + ''' + + self.doctype_name = "Test Newy Child Table" + + try: + doc = frappe.get_doc({ + "doctype": "DocType", + "name": self.doctype_name, + "istable": 1, + "custom": 1, + "module": "Integrations", + "fields": [{ + "label": "Some Field", + "fieldname": "some_fieldname", + "fieldtype": "Data", + "reqd": 1 + }] + }).insert(ignore_permissions=True) + except Exception: + self.fail("Not able to create Child Table Doctype") + + + for column in child_table_fields: + self.assertTrue(frappe.db.has_column(self.doctype_name, column)) + + # check transitioning from child table to normal doctype + doc.istable = 0 + try: + doc.save(ignore_permissions=True) + except Exception: + self.fail("Not able to transition from Child Table Doctype to Normal Doctype") + + self.check_valid_columns(self.assertFalse) + + # check transitioning from normal to child table doctype + doc.istable = 1 + try: + doc.save(ignore_permissions=True) + except Exception: + self.fail("Not able to transition from Normal Doctype to Child Table Doctype") + + self.check_valid_columns(self.assertTrue) + + + def check_valid_columns(self, assertion_method: Callable) -> None: + valid_columns = frappe.get_meta(self.doctype_name).get_valid_columns() + for column in child_table_fields: + assertion_method(column in valid_columns) diff --git a/frappe/tests/test_client.py b/frappe/tests/test_client.py index aed8dc8581..40639e4f98 100644 --- a/frappe/tests/test_client.py +++ b/frappe/tests/test_client.py @@ -101,3 +101,33 @@ class TestClient(unittest.TestCase): execute_cmd, frappe.local.form_dict.cmd ) + + def test_array_values_in_request_args(self): + import requests + from frappe.auth import CookieManager, LoginManager + + frappe.utils.set_request(path="/") + frappe.local.cookie_manager = CookieManager() + frappe.local.login_manager = LoginManager() + frappe.local.login_manager.login_as('Administrator') + params = { + 'doctype': 'DocType', + 'fields': ['name', 'modified'], + 'sid': frappe.session.sid, + } + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + } + url = f'http://{frappe.local.site}:{frappe.conf.webserver_port}/api/method/frappe.client.get_list' + res = requests.post( + url, + json=params, + headers=headers + ) + self.assertEqual(res.status_code, 200) + data = res.json() + first_item = data['message'][0] + self.assertTrue('name' in first_item) + self.assertTrue('modified' in first_item) + frappe.local.login_manager.logout() diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index cdef4354ed..6e96849b35 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -1,21 +1,21 @@ -# -*- coding: utf-8 -*- - -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import datetime +import inspect import unittest from random import choice -import datetime +from unittest.mock import patch import frappe from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.utils import random_string -from frappe.utils.testutils import clear_custom_fields -from frappe.query_builder import Field from frappe.database import savepoint - -from .test_query_builder import run_only_if, db_type_is +from frappe.database.database import Database +from frappe.query_builder import Field from frappe.query_builder.functions import Concat_ws +from frappe.tests.test_query_builder import db_type_is, run_only_if +from frappe.utils import add_days, now, random_string +from frappe.utils.testutils import clear_custom_fields class TestDB(unittest.TestCase): @@ -84,20 +84,6 @@ class TestDB(unittest.TestCase): ), ) - def test_set_value(self): - todo1 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 1')).insert() - todo2 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 2')).insert() - - frappe.db.set_value('ToDo', todo1.name, 'description', 'test_set_value change 1') - self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'test_set_value change 1') - - # multiple set-value - frappe.db.set_value('ToDo', dict(description=('like', '%test_set_value%')), - 'description', 'change 2') - - self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'change 2') - self.assertEqual(frappe.db.get_value('ToDo', todo2.name, 'description'), 'change 2') - def test_escape(self): frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8")) @@ -246,7 +232,6 @@ class TestDB(unittest.TestCase): frappe.delete_doc(test_doctype, doc) clear_custom_fields(test_doctype) - def test_savepoints(self): frappe.db.rollback() save_point = "todonope" @@ -294,6 +279,18 @@ class TestDB(unittest.TestCase): for d in created_docs: self.assertTrue(frappe.db.exists("ToDo", d)) + def test_transaction_writes_error(self): + from frappe.database.database import Database + frappe.db.rollback() + + frappe.db.MAX_WRITES_PER_TRANSACTION = 1 + note = frappe.get_last_doc("ToDo") + note.description = "changed" + with self.assertRaises(frappe.TooManyWritesError) as tmw: + note.save() + + frappe.db.MAX_WRITES_PER_TRANSACTION = Database.MAX_WRITES_PER_TRANSACTION + @run_only_if(db_type_is.MARIADB) class TestDDLCommandsMaria(unittest.TestCase): @@ -353,6 +350,143 @@ class TestDDLCommandsMaria(unittest.TestCase): self.assertEquals(len(indexs_in_table), 2) +class TestDBSetValue(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.todo1 = frappe.get_doc(doctype="ToDo", description="test_set_value 1").insert() + cls.todo2 = frappe.get_doc(doctype="ToDo", description="test_set_value 2").insert() + + def test_update_single_doctype_field(self): + value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + changed_value = not value + + frappe.db.set_value("System Settings", "System Settings", "deny_multiple_sessions", changed_value) + current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + self.assertEqual(current_value, changed_value) + + changed_value = not current_value + frappe.db.set_value("System Settings", None, "deny_multiple_sessions", changed_value) + current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + self.assertEqual(current_value, changed_value) + + changed_value = not current_value + frappe.db.set_single_value("System Settings", "deny_multiple_sessions", changed_value) + current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + self.assertEqual(current_value, changed_value) + + def test_update_single_row_single_column(self): + frappe.db.set_value("ToDo", self.todo1.name, "description", "test_set_value change 1") + updated_value = frappe.db.get_value("ToDo", self.todo1.name, "description") + self.assertEqual(updated_value, "test_set_value change 1") + + def test_update_single_row_multiple_columns(self): + description, status = "Upated by test_update_single_row_multiple_columns", "Closed" + + frappe.db.set_value("ToDo", self.todo1.name, { + "description": description, + "status": status, + }, update_modified=False) + + updated_desciption, updated_status = frappe.db.get_value("ToDo", + filters={"name": self.todo1.name}, + fieldname=["description", "status"] + ) + + self.assertEqual(description, updated_desciption) + self.assertEqual(status, updated_status) + + def test_update_multiple_rows_single_column(self): + frappe.db.set_value("ToDo", {"description": ("like", "%test_set_value%")}, "description", "change 2") + + self.assertEqual(frappe.db.get_value("ToDo", self.todo1.name, "description"), "change 2") + self.assertEqual(frappe.db.get_value("ToDo", self.todo2.name, "description"), "change 2") + + def test_update_multiple_rows_multiple_columns(self): + todos_to_update = frappe.get_all("ToDo", filters={ + "description": ("like", "%test_set_value%"), + "status": ("!=", "Closed") + }, pluck="name") + + frappe.db.set_value("ToDo", { + "description": ("like", "%test_set_value%"), + "status": ("!=", "Closed") + }, { + "status": "Closed", + "priority": "High" + }) + + test_result = frappe.get_all("ToDo", filters={"name": ("in", todos_to_update)}, fields=["status", "priority"]) + + self.assertTrue(all(x for x in test_result if x["status"] == "Closed")) + self.assertTrue(all(x for x in test_result if x["priority"] == "High")) + + def test_update_modified_options(self): + self.todo2.reload() + + todo = self.todo2 + updated_description = f"{todo.description} - by `test_update_modified_options`" + custom_modified = datetime.datetime.fromisoformat(add_days(now(), 10)) + custom_modified_by = "user_that_doesnt_exist@example.com" + + frappe.db.set_value("ToDo", todo.name, "description", updated_description, update_modified=False) + self.assertEqual(updated_description, frappe.db.get_value("ToDo", todo.name, "description")) + self.assertEqual(todo.modified, frappe.db.get_value("ToDo", todo.name, "modified")) + + frappe.db.set_value("ToDo", todo.name, "description", "test_set_value change 1", modified=custom_modified, modified_by=custom_modified_by) + self.assertTupleEqual( + (custom_modified, custom_modified_by), + frappe.db.get_value("ToDo", todo.name, ["modified", "modified_by"]) + ) + + def test_for_update(self): + self.todo1.reload() + + with patch.object(Database, "sql") as sql_called: + frappe.db.set_value( + self.todo1.doctype, + self.todo1.name, + "description", + f"{self.todo1.description}-edit by `test_for_update`" + ) + first_query = sql_called.call_args_list[0].args[0] + second_query = sql_called.call_args_list[1].args[0] + + self.assertTrue(sql_called.call_count == 2) + self.assertTrue("FOR UPDATE" in first_query) + if frappe.conf.db_type == "postgres": + from frappe.database.postgres.database import modify_query + self.assertTrue(modify_query("UPDATE `tabToDo` SET") in second_query) + if frappe.conf.db_type == "mariadb": + self.assertTrue("UPDATE `tabToDo` SET" in second_query) + + def test_cleared_cache(self): + self.todo2.reload() + + with patch.object(frappe, "clear_document_cache") as clear_cache: + frappe.db.set_value( + self.todo2.doctype, + self.todo2.name, + "description", + f"{self.todo2.description}-edit by `test_cleared_cache`" + ) + clear_cache.assert_called() + + def test_update_alias(self): + args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`") + kwargs = {"for_update": False, "modified": None, "modified_by": None, "update_modified": True, "debug": False} + + self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update)) + + with patch.object(Database, "set_value") as set_value: + frappe.db.update(*args, **kwargs) + set_value.assert_called_once() + set_value.assert_called_with(*args, **kwargs) + + @classmethod + def tearDownClass(cls): + frappe.db.rollback() + + @run_only_if(db_type_is.POSTGRES) class TestDDLCommandsPost(unittest.TestCase): test_table_name = "TestNotes" diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index d2c54ef18c..66eb05391a 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -103,10 +103,7 @@ def get_other_fields_meta(meta): default_fields_map = { 'name': ('Data', 0), 'owner': ('Data', 0), - 'parent': ('Data', 0), - 'parentfield': ('Data', 0), 'modified_by': ('Data', 0), - 'parenttype': ('Data', 0), 'creation': ('Datetime', 0), 'modified': ('Datetime', 0), 'idx': ('Int', 8), @@ -117,8 +114,12 @@ def get_other_fields_meta(meta): if meta.track_seen: optional_fields.append('_seen') + child_table_fields_map = {} + if meta.istable: + child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS}) + optional_fields_map = {field: ('Text', 0) for field in optional_fields} - fields = dict(default_fields_map, **optional_fields_map) + fields = dict(default_fields_map, **optional_fields_map, **child_table_fields_map) field_map = [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()] return field_map diff --git a/frappe/tests/test_docstatus.py b/frappe/tests/test_docstatus.py new file mode 100644 index 0000000000..7692bca48b --- /dev/null +++ b/frappe/tests/test_docstatus.py @@ -0,0 +1,26 @@ +import unittest + +from frappe.model.docstatus import DocStatus + + +class TestDocStatus(unittest.TestCase): + def test_draft(self): + self.assertEqual(DocStatus(0), DocStatus.draft()) + + self.assertTrue(DocStatus.draft().is_draft()) + self.assertFalse(DocStatus.draft().is_cancelled()) + self.assertFalse(DocStatus.draft().is_submitted()) + + def test_submitted(self): + self.assertEqual(DocStatus(1), DocStatus.submitted()) + + self.assertFalse(DocStatus.submitted().is_draft()) + self.assertTrue(DocStatus.submitted().is_submitted()) + self.assertFalse(DocStatus.submitted().is_cancelled()) + + def test_cancelled(self): + self.assertEqual(DocStatus(2), DocStatus.cancelled()) + + self.assertFalse(DocStatus.cancelled().is_draft()) + self.assertFalse(DocStatus.cancelled().is_submitted()) + self.assertTrue(DocStatus.cancelled().is_cancelled()) diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index ef9515f5ba..ad9f8fdd11 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -3,6 +3,8 @@ import unittest, frappe, re, email +from frappe.email.doctype.email_account.test_email_account import TestEmailAccount + test_dependencies = ['Email Account'] class TestEmail(unittest.TestCase): @@ -173,12 +175,35 @@ class TestEmail(unittest.TestCase): frappe.db.delete("Communication", {"sender": "sukh@yyy.com"}) with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw: - mails = email_account.get_inbound_mails(test_mails=[raw.read()]) + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + raw.read() + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + changed_flag = False + if not email_account.enable_incoming: + email_account.enable_incoming = True + changed_flag = True + mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages) + + # mails = email_account.get_inbound_mails(test_mails=[raw.read()]) communication = mails[0].process() self.assertTrue(re.search(''']*src=["']/private/files/rtco1.png[^>]*>''', communication.content)) self.assertTrue(re.search(''']*src=["']/private/files/rtco2.png[^>]*>''', communication.content)) + if changed_flag: + email_account.enable_incoming = False + if __name__ == '__main__': frappe.connect() diff --git a/frappe/tests/test_form_load.py b/frappe/tests/test_form_load.py index d59e8f1570..b10d5eb796 100644 --- a/frappe/tests/test_form_load.py +++ b/frappe/tests/test_form_load.py @@ -1,9 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import frappe, unittest -from frappe.desk.form.load import getdoctype, getdoc +from frappe.desk.form.load import getdoctype, getdoc, get_docinfo from frappe.core.page.permission_manager.permission_manager import update, reset, add from frappe.custom.doctype.property_setter.property_setter import make_property_setter +from frappe.utils.file_manager import save_file test_dependencies = ['Blog Category', 'Blogger'] @@ -141,9 +142,52 @@ class TestFormLoad(unittest.TestCase): contact.delete() + def test_get_doc_info(self): + note = frappe.new_doc("Note") + note.content = "some content" + note.title = frappe.generate_hash(length=20) + note.insert() + + note.content = "new content" + # trigger a version + note.save(ignore_version=False) + + note.add_comment(text="test") + + note.add_tag("test_tag") + note.add_tag("more_tag") + + # empty attachment + save_file("test_file", b"", note.doctype, note.name, decode=True) + + frappe.get_doc({ + "doctype": "Communication", + "communication_type": "Communication", + "content": "test email", + "reference_doctype": note.doctype, + "reference_name": note.name, + }).insert() + + + docinfo = get_docinfo(note) + + self.assertEqual(len(docinfo.comments), 1) + self.assertIn("test", docinfo.comments[0].content) + + self.assertGreaterEqual(len(docinfo.versions), 2) + + self.assertEqual(set(docinfo.tags.split(",")), {"more_tag", "test_tag"}) + + self.assertEqual(len(docinfo.attachments), 1) + self.assertIn("test_file", docinfo.attachments[0].file_name) + + self.assertEqual(len(docinfo.communications), 1) + self.assertIn("email", docinfo.communications[0].content) + note.delete() + def get_blog(blog_name): frappe.response.docs = [] getdoc('Blog Post', blog_name) doc = frappe.response.docs[0] - return doc \ No newline at end of file + return doc diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 3031d3e344..2309d8fc2b 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -144,6 +144,7 @@ class TestNaming(unittest.TestCase): current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0] self.assertEqual(current_index.get('current'), 2) + frappe.db.delete("Series", {"name": series}) def test_naming_for_cancelled_and_amended_doc(self): @@ -166,25 +167,20 @@ class TestNaming(unittest.TestCase): doc.submit() doc.cancel() cancelled_name = doc.name - self.assertEqual(cancelled_name, "{}-CANC-0".format(original_name)) + self.assertEqual(cancelled_name, original_name) amended_doc = frappe.copy_doc(doc) amended_doc.docstatus = 0 amended_doc.amended_from = doc.name amended_doc.save() - self.assertEqual(amended_doc.name, original_name) + self.assertEqual(amended_doc.name, "{}-1".format(original_name)) amended_doc.submit() amended_doc.cancel() - self.assertEqual(amended_doc.name, "{}-CANC-1".format(original_name)) + self.assertEqual(amended_doc.name, "{}-1".format(original_name)) submittable_doctype.delete() - def test_parse_naming_series_for_consecutive_week_number(self): - week = determine_consecutive_week_number(now_datetime()) - name = parse_naming_series('PREFIX-.WW.-SUFFIX') - expected_name = 'PREFIX-{}-SUFFIX'.format(week) - self.assertEqual(name, expected_name) def test_determine_consecutive_week_number(self): from datetime import datetime @@ -207,4 +203,4 @@ class TestNaming(unittest.TestCase): dt = datetime.fromisoformat("2021-12-31") w = determine_consecutive_week_number(dt) - self.assertEqual(w, "52") + self.assertEqual(w, "52") \ No newline at end of file diff --git a/frappe/tests/test_patches.py b/frappe/tests/test_patches.py index 64e8684f55..32e7b7ff3a 100644 --- a/frappe/tests/test_patches.py +++ b/frappe/tests/test_patches.py @@ -38,6 +38,16 @@ execute:frappe.function(arg="1") app.module.patch3 """ +COMMENTED_OUT = """ +[pre_model_sync] +app.module.patch1 +# app.module.patch2 # rerun +app.module.patch3 + +[post_model_sync] +app.module.patch4 +""" + class TestPatches(unittest.TestCase): def test_patch_module_names(self): frappe.flags.final_patches = [] @@ -70,50 +80,55 @@ class TestPatches(unittest.TestCase): class TestPatchReader(unittest.TestCase): + + def get_patches(self): + return ( + patch_handler.get_patches_from_app("frappe"), + patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.pre_model_sync), + patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.post_model_sync) + ) + @patch("builtins.open", new_callable=mock_open, read_data=EMTPY_FILE) def test_empty_file(self, _file): - all_patches = patch_handler.get_patches_from_app("frappe") - pre = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.pre_model_sync) - post = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.post_model_sync) - self.assertEqual(all_patches, []) + all, pre, post = self.get_patches() + self.assertEqual(all, []) self.assertEqual(pre, []) self.assertEqual(post, []) @patch("builtins.open", new_callable=mock_open, read_data=EMTPY_SECTION) def test_empty_sections(self, _file): - all_patches = patch_handler.get_patches_from_app("frappe") - pre = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.pre_model_sync) - post = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.post_model_sync) - self.assertEqual(all_patches, []) + all, pre, post = self.get_patches() + self.assertEqual(all, []) self.assertEqual(pre, []) self.assertEqual(post, []) @patch("builtins.open", new_callable=mock_open, read_data=FILLED_SECTIONS) def test_new_style(self, _file): - all_patches = patch_handler.get_patches_from_app("frappe") - pre = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.pre_model_sync) - post = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.post_model_sync) - self.assertEqual(all_patches, ["app.module.patch1", "app.module.patch2", "app.module.patch3"]) + all, pre, post = self.get_patches() + self.assertEqual(all, ["app.module.patch1", "app.module.patch2", "app.module.patch3"]) self.assertEqual(pre, ["app.module.patch1", "app.module.patch2"]) self.assertEqual(post, ["app.module.patch3",]) @patch("builtins.open", new_callable=mock_open, read_data=OLD_STYLE_PATCH_TXT) def test_old_style(self, _file): - all_patches = patch_handler.get_patches_from_app("frappe") - pre = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.pre_model_sync) - post = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.post_model_sync) - self.assertEqual(all_patches, ["app.module.patch1", "app.module.patch2", "app.module.patch3"]) + all, pre, post = self.get_patches() + self.assertEqual(all, ["app.module.patch1", "app.module.patch2", "app.module.patch3"]) self.assertEqual(pre, ["app.module.patch1", "app.module.patch2", "app.module.patch3"]) self.assertEqual(post, []) @patch("builtins.open", new_callable=mock_open, read_data=EDGE_CASES) def test_new_style_edge_cases(self, _file): - pre = patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.pre_model_sync) + all, pre, post = self.get_patches() self.assertEqual(pre, [ "App.module.patch1", "app.module.patch2 # rerun", 'execute:frappe.db.updatedb("Item")', 'execute:frappe.function(arg="1")', ]) + + @patch("builtins.open", new_callable=mock_open, read_data=COMMENTED_OUT) + def test_ignore_comments(self, _file): + all, pre, post = self.get_patches() + self.assertEqual(pre, ["app.module.patch1", "app.module.patch3"]) diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index d2242cc6f7..ea700b183e 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -5,7 +5,7 @@ import frappe from frappe.query_builder.custom import ConstantColumn from frappe.query_builder.functions import Coalesce, GroupConcat, Match from frappe.query_builder.utils import db_type_is - +from frappe.query_builder import Case def run_only_if(dbtype: db_type_is) -> Callable: return unittest.skipIf( @@ -25,8 +25,14 @@ class TestCustomFunctionsMariaDB(unittest.TestCase): ) def test_constant_column(self): - query = frappe.qb.from_("DocType").select("name", ConstantColumn("John").as_("User")) - self.assertEqual(query.get_sql(), "SELECT `name`,'John' `User` FROM `tabDocType`") + query = frappe.qb.from_("DocType").select( + "name", ConstantColumn("John").as_("User") + ) + self.assertEqual( + query.get_sql(), "SELECT `name`,'John' `User` FROM `tabDocType`" + ) + + @run_only_if(db_type_is.POSTGRES) class TestCustomFunctionsPostgres(unittest.TestCase): def test_concat(self): @@ -39,8 +45,13 @@ class TestCustomFunctionsPostgres(unittest.TestCase): ) def test_constant_column(self): - query = frappe.qb.from_("DocType").select("name", ConstantColumn("John").as_("User")) - self.assertEqual(query.get_sql(), 'SELECT "name",\'John\' "User" FROM "tabDocType"') + query = frappe.qb.from_("DocType").select( + "name", ConstantColumn("John").as_("User") + ) + self.assertEqual( + query.get_sql(), 'SELECT "name",\'John\' "User" FROM "tabDocType"' + ) + class TestBuilderBase(object): def test_adding_tabs(self): @@ -55,23 +66,95 @@ class TestBuilderBase(object): self.assertIsInstance(query.run, Callable) self.assertIsInstance(data, list) - def test_walk(self): - DocType = frappe.qb.DocType('DocType') + +class TestParameterization(unittest.TestCase): + def test_where_conditions(self): + DocType = frappe.qb.DocType("DocType") query = ( frappe.qb.from_(DocType) .select(DocType.name) - .where((DocType.owner == "Administrator' --") - & (Coalesce(DocType.search_fields == "subject")) - ) + .where((DocType.owner == "Administrator' --")) ) self.assertTrue("walk" in dir(query)) query, params = query.walk() self.assertIn("%(param1)s", query) - self.assertIn("%(param2)s", query) - self.assertIn("param1",params) - self.assertEqual(params["param1"],"Administrator' --") - self.assertEqual(params["param2"],"subject") + self.assertIn("param1", params) + self.assertEqual(params["param1"], "Administrator' --") + + def test_set_cnoditions(self): + DocType = frappe.qb.DocType("DocType") + query = frappe.qb.update(DocType).set(DocType.value, "some_value") + + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("param1", params) + self.assertEqual(params["param1"], "some_value") + + def test_where_conditions_functions(self): + DocType = frappe.qb.DocType("DocType") + query = ( + frappe.qb.from_(DocType) + .select(DocType.name) + .where(Coalesce(DocType.search_fields == "subject")) + ) + + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("param1", params) + self.assertEqual(params["param1"], "subject") + + def test_case(self): + DocType = frappe.qb.DocType("DocType") + query = ( + frappe.qb.from_(DocType) + .select( + Case() + .when(DocType.search_fields == "value", "other_value") + .when(Coalesce(DocType.search_fields == "subject_in_function"), "true_value") + .else_("Overdue") + ) + ) + + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("param1", params) + self.assertEqual(params["param1"], "value") + self.assertEqual(params["param2"], "other_value") + self.assertEqual(params["param3"], "subject_in_function") + self.assertEqual(params["param4"], "true_value") + self.assertEqual(params["param5"], "Overdue") + + def test_case_in_update(self): + DocType = frappe.qb.DocType("DocType") + query = ( + frappe.qb.update(DocType) + .set( + "parent", + Case() + .when(DocType.search_fields == "value", "other_value") + .when(Coalesce(DocType.search_fields == "subject_in_function"), "true_value") + .else_("Overdue") + ) + ) + + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("param1", params) + self.assertEqual(params["param1"], "value") + self.assertEqual(params["param2"], "other_value") + self.assertEqual(params["param3"], "subject_in_function") + self.assertEqual(params["param4"], "true_value") + self.assertEqual(params["param5"], "Overdue") + @run_only_if(db_type_is.MARIADB) @@ -84,6 +167,7 @@ class TestBuilderMaria(unittest.TestCase, TestBuilderBase): "SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql() ) + @run_only_if(db_type_is.POSTGRES) class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 8ac6218b5e..27d1f7651d 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1,22 +1,28 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest -import frappe -from frappe.utils import evaluate_filters, money_in_words, scrub_urls, get_url -from frappe.utils import validate_url, validate_email_address -from frappe.utils import ceil, floor -from frappe.utils.data import cast, validate_python_code -from frappe.utils.diff import get_version_diff, version_query, _get_value_from_version - -from PIL import Image -from frappe.utils.image import strip_exif_data, optimize_image import io +import json +import unittest +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from enum import Enum from mimetypes import guess_type -from datetime import datetime, timedelta, date - from unittest.mock import patch +import pytz +from PIL import Image + +import frappe +from frappe.utils import ceil, evaluate_filters, floor, format_timedelta +from frappe.utils import get_url, money_in_words, parse_timedelta, scrub_urls +from frappe.utils import validate_email_address, validate_url +from frappe.utils.data import cast, get_time, get_timedelta, nowtime, now_datetime, validate_python_code +from frappe.utils.diff import _get_value_from_version, get_version_diff, version_query +from frappe.utils.image import optimize_image, strip_exif_data +from frappe.utils.response import json_handler + + class TestFilters(unittest.TestCase): def test_simple_dict(self): self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open'}, {'status': 'Open'})) @@ -273,9 +279,7 @@ class TestPythonExpressions(unittest.TestCase): for expr in invalid_expressions: self.assertRaises(frappe.ValidationError, validate_python_code, expr) - class TestDiffUtils(unittest.TestCase): - @classmethod def setUpClass(cls): cls.doc = frappe.get_doc(doctype="Client Script", dt="Client Script") @@ -330,8 +334,85 @@ class TestDateUtils(unittest.TestCase): self.assertEqual(frappe.utils.get_last_day_of_week("2020-12-28"), frappe.utils.getdate("2021-01-02")) -class TestXlsxUtils(unittest.TestCase): + def test_get_time(self): + datetime_input = now_datetime() + timedelta_input = get_timedelta() + time_input = nowtime() + self.assertIsInstance(get_time(datetime_input), time) + self.assertIsInstance(get_time(timedelta_input), time) + self.assertIsInstance(get_time(time_input), time) + self.assertIsInstance(get_time("100:2:12"), time) + self.assertIsInstance(get_time(str(datetime_input)), time) + self.assertIsInstance(get_time(str(timedelta_input)), time) + self.assertIsInstance(get_time(str(time_input)), time) + + def test_get_timedelta(self): + datetime_input = now_datetime() + timedelta_input = get_timedelta() + time_input = nowtime() + + self.assertIsInstance(get_timedelta(), timedelta) + self.assertIsInstance(get_timedelta("100:2:12"), timedelta) + self.assertIsInstance(get_timedelta("17:21:00"), timedelta) + self.assertIsInstance(get_timedelta("2012-01-19 17:21:00"), timedelta) + self.assertIsInstance(get_timedelta(str(datetime_input)), timedelta) + self.assertIsInstance(get_timedelta(str(timedelta_input)), timedelta) + self.assertIsInstance(get_timedelta(str(time_input)), timedelta) + +class TestResponse(unittest.TestCase): + def test_json_handler(self): + class TEST(Enum): + ABC = "!@)@)!" + BCE = "ENJD" + + GOOD_OBJECT = { + "time_types": [ + date(year=2020, month=12, day=2), + datetime(year=2020, month=12, day=2, hour=23, minute=23, second=23, microsecond=23, tzinfo=pytz.utc), + time(hour=23, minute=23, second=23, microsecond=23, tzinfo=pytz.utc), + timedelta(days=10, hours=12, minutes=120, seconds=10), + ], + "float": [ + Decimal(29.21), + ], + "doc": [ + frappe.get_doc("System Settings"), + ], + "iter": [ + {1, 2, 3}, + (1, 2, 3), + "abcdef", + ], + "string": "abcdef" + } + + BAD_OBJECT = {"Enum": TEST} + + processed_object = json.loads(json.dumps(GOOD_OBJECT, default=json_handler)) + + self.assertTrue(all([isinstance(x, str) for x in processed_object["time_types"]])) + self.assertTrue(all([isinstance(x, float) for x in processed_object["float"]])) + self.assertTrue(all([isinstance(x, (list, str)) for x in processed_object["iter"]])) + self.assertIsInstance(processed_object["string"], str) + with self.assertRaises(TypeError): + json.dumps(BAD_OBJECT, default=json_handler) + +class TestTimeDeltaUtils(unittest.TestCase): + def test_format_timedelta(self): + self.assertEqual(format_timedelta(timedelta(seconds=0)), "0:00:00") + self.assertEqual(format_timedelta(timedelta(hours=10)), "10:00:00") + self.assertEqual(format_timedelta(timedelta(hours=100)), "100:00:00") + self.assertEqual(format_timedelta(timedelta(seconds=100, microseconds=129)), "0:01:40.000129") + self.assertEqual(format_timedelta(timedelta(seconds=100, microseconds=12212199129)), "3:25:12.199129") + + def test_parse_timedelta(self): + self.assertEqual(parse_timedelta("0:0:0"), timedelta(seconds=0)) + self.assertEqual(parse_timedelta("10:0:0"), timedelta(hours=10)) + self.assertEqual(parse_timedelta("7 days, 0:32:18.192221"), timedelta(days=7, seconds=1938, microseconds=192221)) + self.assertEqual(parse_timedelta("7 days, 0:32:18"), timedelta(days=7, seconds=1938)) + +class TestXlsxUtils(unittest.TestCase): def test_unescape(self): from frappe.utils.xlsxutils import handle_html diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index e40a07c0ec..97b9fc9b67 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -1,7 +1,9 @@ import unittest +from unittest.mock import patch import frappe from frappe.utils import set_request +from frappe.website.page_renderers.static_page import StaticPage from frappe.website.serve import get_response, get_response_content from frappe.website.utils import (build_response, clear_website_cache, get_home_page) @@ -20,6 +22,7 @@ class TestWebsite(unittest.TestCase): doctype='User', email='test-user-for-home-page@example.com', first_name='test')).insert(ignore_if_duplicate=True) + user.reload() role = frappe.get_doc(dict( doctype = 'Role', @@ -96,6 +99,19 @@ class TestWebsite(unittest.TestCase): response = get_response() self.assertEqual(response.status_code, 200) + set_request(method="GET", path="/_test/assets/image.jpg") + response = get_response() + self.assertEqual(response.status_code, 200) + + set_request(method="GET", path="/_test/assets/image") + response = get_response() + self.assertEqual(response.status_code, 200) + + with patch.object(StaticPage, "render") as static_render: + set_request(method="GET", path="/_test/assets/image") + response = get_response() + static_render.assert_called() + def test_error_page(self): set_request(method='GET', path='/_test/problematic_page') response = get_response() @@ -126,7 +142,6 @@ class TestWebsite(unittest.TestCase): response = get_response() self.assertEqual(response.status_code, 404) - def test_redirect(self): import frappe.hooks frappe.set_user('Administrator') diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index 0a541343f3..f682a51e17 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -151,6 +151,7 @@ My Account,Mein Konto, New Address,Neue Adresse, New Contact,Neuer Kontakt, Next,Weiter, +No,Nein, No Data,Keine Daten, No address added yet.,Noch keine Adresse hinzugefügt., No contacts added yet.,Noch keine Kontakte hinzugefügt., @@ -349,7 +350,7 @@ Add a New Role,Neue Rolle hinzufügen, Add a column,Spalte einfügen, Add a comment,Einen Kommentar hinzufügen, Add a new section,Fügen Sie einen neuen Abschnitt hinzu, -Add a tag ...,Füge einen Tag hinzu ..., +Add a tag ...,Füge ein Schlagwort hinzu ..., Add all roles,Alle Rollen hinzufügen, Add custom forms.,Benutzerdefinierte Formulare hinzufügen, Add custom javascript to forms.,Benutzerdefiniertes Javascript zum Formular hinzufügen, @@ -946,6 +947,7 @@ Edit Auto Email Report Settings,Bearbeiten Sie die Einstellungen für automatisc Edit Custom HTML,Benutzerdefiniertes HTML bearbeiten, Edit DocType,DocType bearbeiten, Edit Filter,Filter bearbeiten, +Edit Filters,Filter bearbeiten, Edit Format,Format bearbeiten, Edit HTML,HTML bearbeiten, Edit Heading,Kopf bearbeiten, @@ -1230,6 +1232,7 @@ Hide Copy,Kopie ausblenden, Hide Footer Signup,Fußzeilen-Anmeldung ausblenden, Hide Sidebar and Menu,Seitenleiste und Menü ausblenden, Hide Standard Menu,Standardmenü ausblenden, +Hide Tags,Schlagworte ausblenden, Hide Weekends,Wochenenden ausblenden, Hide details,Details ausblenden, Hide footer in auto email reports,Fußzeile in automatischen E-Mail-Berichten ausblenden, @@ -1573,6 +1576,7 @@ Module Def,Modul-Def, Module Name,Modulname, Module Not Found,Modul nicht gefunden, Module Path,Modulpfad, +Module Profile, Modulprofil, Module to Export,Module für den Export, Modules HTML,Modul-HTML, Monospace,Monospace, @@ -1649,7 +1653,7 @@ No Preview,Keine Vorschau, No Preview Available,Keine Vorschau vorhanden, No Printer is Available.,Es ist kein Drucker verfügbar., No Results,Keine Ergebnisse, -No Tags,No Tags, +No Tags,Keine Schlagworte, No alerts for today,Keine Warnungen für heute, No comments yet,Noch keine Kommentare, No comments yet. Start a new discussion.,Noch keine Kommentare. Starten Sie eine neue Diskussion., @@ -2039,7 +2043,7 @@ Remove,Entfernen, Remove Field,Feld entfernen, Remove Filter,Filter entfernen, Remove Section,Abschnitt entfernen, -Remove Tag,Markierung entfernen, +Remove Tag,Schlagwort entfernen, Remove all customizations?,Alle Anpassungen entfernen?, Removed {0},{0} entfernt, Rename many items by uploading a .csv file.,Viele Elemente auf einmal umbenennen durch Hochladen einer .CSV-Datei, @@ -3249,7 +3253,7 @@ DocType Action,DocType-Aktion, DocType Event,DocType-Ereignis, DocType Link,DocType Link, Document Share,Dokumentenfreigabe, -Document Tag,Dokument-Tag, +Document Tag,Dokument-Schlagwort, Document Title,Dokumenttitel, Document Type Field Mapping,Dokumenttyp-Feldzuordnung, Document Type Mapping,Dokumenttypzuordnung, @@ -3557,9 +3561,9 @@ Skipping column {0},Spalte {0} wird übersprungen, Social Home,Soziales Zuhause, Some columns might get cut off when printing to PDF. Try to keep number of columns under 10.,"Einige Spalten werden beim Drucken in PDF möglicherweise abgeschnitten. Versuchen Sie, die Anzahl der Spalten unter 10 zu halten.", Something went wrong during the token generation. Click on {0} to generate a new one.,"Während der Token-Generierung ist ein Fehler aufgetreten. Klicken Sie auf {0}, um eine neue zu erstellen.", -Submit After Import,Nach dem Import einreichen, -Submitting...,Einreichen ..., -Success! You are good to go 👍,Erfolg! Du bist gut zu gehen 👍, +Submit After Import,Nach dem Import buchen, +Submitting...,wird verbucht..., +Success! You are good to go 👍,Erfolg! Du kannst nun durchstarten 👍, Successful Transactions,Erfolgreiche Transaktionen, Successfully Submitted!,Erfolgreich eingereicht!, Successfully imported {0} record.,{0} Datensatz erfolgreich importiert., @@ -3571,9 +3575,9 @@ Sync Contacts,Kontakte synchronisieren, Sync with Google Calendar,Mit Google Kalender synchronisieren, Sync with Google Contacts,Mit Google-Kontakten synchronisieren, Synced,Synchronisiert, -Syncing,Synchronisierung, +Syncing,Synchronisiert, Syncing {0} of {1},{0} von {1} synchronisieren, -Tag Link,Tag-Link, +Tag Link,Schlagwortverknüpfung, Take Backup,Backup erstellen, Template Error,Vorlagenfehler, Template Options,Vorlagenoptionen, @@ -3795,7 +3799,7 @@ Start,Start, Start Time,Startzeit, Status,Status, Submitted,Gebucht, -Tag,Etikett, +Tag,Schlagwort, Template,Vorlage, Thursday,Donnerstag, Title,Bezeichnung, @@ -4027,7 +4031,7 @@ Please select target language for translation,Bitte wählen Sie die Zielsprache Select Language,Sprache auswählen, Confirm Translations,Übersetzungen bestätigen, Contributed Translations,Beigetragene Übersetzungen, -Show Tags,Tags anzeigen, +Show Tags,Schlagworte anzeigen, Do not have permission to access {0} bucket.,Sie haben keine Berechtigung zum Zugriff auf den Bucket {0}., Allow document creation via Email,Dokumenterstellung per E-Mail zulassen, Sender Field,Absenderfeld, diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 9deec0a77c..141adb9ea6 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import functools @@ -24,9 +24,6 @@ import frappe from frappe.utils.data import * from frappe.utils.html_utils import sanitize_html -default_fields = ['doctype', 'name', 'owner', 'creation', 'modified', 'modified_by', - 'parent', 'parentfield', 'parenttype', 'idx', 'docstatus'] - def get_fullname(user=None): """get the full name (first name + last name) of the user from User""" @@ -56,8 +53,8 @@ def get_email_address(user=None): def get_formatted_email(user, mail=None): """get Email Address of user formatted as: `John Doe `""" fullname = get_fullname(user) - method = get_hook_method('get_sender_details') + if method: sender_name, mail = method() # if method exists but sender_name is "" diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 891a55deda..50c71bdc2e 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1,17 +1,22 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from typing import Optional -import frappe -import operator -import json import base64 -import re, datetime, math, time +import datetime +import json +import math +import operator +import re +import time from code import compile_command -from urllib.parse import quote, urljoin -from frappe.desk.utils import slug -from click import secho from enum import Enum +from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.parse import quote, urljoin + +from click import secho + +import frappe +from frappe.desk.utils import slug DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S.%f" @@ -99,11 +104,17 @@ def get_timedelta(time: Optional[str] = None) -> Optional[datetime.timedelta]: datetime.timedelta: Timedelta object equivalent of the passed `time` string """ from dateutil import parser + from dateutil.parser import ParserError time = time or "0:0:0" try: - t = parser.parse(time) + try: + t = parser.parse(time) + except ParserError as e: + if "day" in e.args[1] or "hour must be in" in e.args[0]: + return parse_timedelta(time) + raise e return datetime.timedelta( hours=t.hour, minutes=t.minute, seconds=t.second, microseconds=t.microsecond ) @@ -201,7 +212,7 @@ def get_time_zone(): return frappe.cache().get_value("time_zone", _get_time_zone) def convert_utc_to_timezone(utc_timestamp, time_zone): - from pytz import timezone, UnknownTimeZoneError + from pytz import UnknownTimeZoneError, timezone utcnow = timezone('UTC').localize(utc_timestamp) try: return utcnow.astimezone(timezone(time_zone)) @@ -318,17 +329,24 @@ def get_year_ending(date): # last day of this month return add_to_date(date, days=-1) -def get_time(time_str): +def get_time(time_str: str) -> datetime.time: from dateutil import parser + from dateutil.parser import ParserError if isinstance(time_str, datetime.datetime): return time_str.time() elif isinstance(time_str, datetime.time): return time_str - else: - if isinstance(time_str, datetime.timedelta): - time_str = str(time_str) + elif isinstance(time_str, datetime.timedelta): + return (datetime.datetime.min + time_str).time() + try: return parser.parse(time_str).time() + except ParserError as e: + if "day" in e.args[1] or "hour must be in" in e.args[0]: + return ( + datetime.datetime.min + parse_timedelta(time_str) + ).time() + raise e def get_datetime_str(datetime_obj): if isinstance(datetime_obj, str): @@ -610,7 +628,7 @@ def cast(fieldtype, value=None): value = flt(value) elif fieldtype in ("Int", "Check"): - value = cint(value) + value = cint(sbool(value)) elif fieldtype in ("Data", "Text", "Small Text", "Long Text", "Text Editor", "Select", "Link", "Dynamic Link"): @@ -726,7 +744,7 @@ def ceil(s): def cstr(s, encoding='utf-8'): return frappe.as_unicode(s, encoding) -def sbool(x): +def sbool(x: str) -> Union[bool, Any]: """Converts str object to Boolean if possible. Example: "true" becomes True @@ -737,12 +755,15 @@ def sbool(x): x (str): String to be converted to Bool Returns: - object: Returns Boolean or type(x) + object: Returns Boolean or x """ - from distutils.util import strtobool - try: - return bool(strtobool(x)) + val = x.lower() + if val in ('true', '1'): + return True + elif val in ('false', '0'): + return False + return x except Exception: return x @@ -917,13 +938,13 @@ number_format_info = { "#.########": (".", "", 8) } -def get_number_format_info(format): +def get_number_format_info(format: str) -> Tuple[str, str, int]: return number_format_info.get(format) or (".", ",", 2) # # convert currency to words # -def money_in_words(number, main_currency = None, fraction_currency=None): +def money_in_words(number: str, main_currency: Optional[str] = None, fraction_currency: Optional[str] = None): """ Returns string in words with currency and fraction currency. """ @@ -1009,9 +1030,11 @@ def is_image(filepath): def get_thumbnail_base64_for_image(src): from os.path import exists as file_exists + from PIL import Image + + from frappe import cache, safe_decode from frappe.core.doctype.file.file import get_local_image - from frappe import safe_decode, cache if not src: frappe.throw('Invalid source for image: {0}'.format(src)) @@ -1302,7 +1325,7 @@ operator_map = { "None": lambda a, b: (not a) and True or False } -def evaluate_filters(doc, filters): +def evaluate_filters(doc, filters: Union[Dict, List, Tuple]): '''Returns true if doc matches filters''' if isinstance(filters, dict): for key, value in filters.items(): @@ -1319,7 +1342,7 @@ def evaluate_filters(doc, filters): return True -def compare(val1, condition, val2, fieldtype=None): +def compare(val1: Any, condition: str, val2: Any, fieldtype: Optional[str] = None): ret = False if fieldtype: val2 = cast(fieldtype, val2) @@ -1328,7 +1351,7 @@ def compare(val1, condition, val2, fieldtype=None): return ret -def get_filter(doctype, f, filters_config=None): +def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) -> "frappe._dict": """Returns a _dict like { @@ -1339,7 +1362,7 @@ def get_filter(doctype, f, filters_config=None): "fieldtype": } """ - from frappe.model import default_fields, optional_fields + from frappe.model import default_fields, optional_fields, child_table_fields if isinstance(f, dict): key, value = next(iter(f.items())) @@ -1377,7 +1400,7 @@ def get_filter(doctype, f, filters_config=None): frappe.throw(frappe._("Operator must be one of {0}").format(", ".join(valid_operators))) - if f.doctype and (f.fieldname not in default_fields + optional_fields): + if f.doctype and (f.fieldname not in default_fields + optional_fields + child_table_fields): # verify fieldname belongs to the doctype meta = frappe.get_meta(f.doctype) if not meta.has_field(f.fieldname): @@ -1415,8 +1438,10 @@ def make_filter_dict(filters): return _filter def sanitize_column(column_name): - from frappe import _ import sqlparse + + from frappe import _ + regex = re.compile("^.*[,'();].*") column_name = sqlparse.format(column_name, strip_comments=True, keyword_case="lower") blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'and', 'or'] @@ -1492,9 +1517,10 @@ def strip(val, chars=None): return (val or "").replace("\ufeff", "").replace("\u200b", "").strip(chars) def to_markdown(html): - from html2text import html2text from html.parser import HTMLParser + from html2text import html2text + text = None try: text = html2text(html or '') @@ -1504,7 +1530,8 @@ def to_markdown(html): return text def md_to_html(markdown_text): - from markdown2 import markdown as _markdown, MarkdownError + from markdown2 import MarkdownError + from markdown2 import markdown as _markdown extras = { 'fenced-code-blocks': None, @@ -1529,14 +1556,14 @@ def md_to_html(markdown_text): def markdown(markdown_text): return md_to_html(markdown_text) -def is_subset(list_a, list_b): +def is_subset(list_a: List, list_b: List) -> bool: '''Returns whether list_a is a subset of list_b''' return len(list(set(list_a) & set(list_b))) == len(list_a) -def generate_hash(*args, **kwargs): +def generate_hash(*args, **kwargs) -> str: return frappe.generate_hash(*args, **kwargs) -def guess_date_format(date_string): +def guess_date_format(date_string: str) -> str: DATE_FORMATS = [ r"%d/%b/%y", r"%d-%m-%Y", @@ -1611,13 +1638,13 @@ def guess_date_format(date_string): if date_format and time_format: return (date_format + ' ' + time_format).strip() -def validate_json_string(string): +def validate_json_string(string: str) -> None: try: json.loads(string) except (TypeError, ValueError): raise frappe.ValidationError -def get_user_info_for_avatar(user_id): +def get_user_info_for_avatar(user_id: str) -> Dict: user_info = { "email": user_id, "image": "", @@ -1664,3 +1691,30 @@ class UnicodeWithAttrs(str): def __init__(self, text): self.toc_html = text.toc_html self.metadata = text.metadata + + +def format_timedelta(o: datetime.timedelta) -> str: + # mariadb allows a wide diff range - https://mariadb.com/kb/en/time/ + # but frappe doesnt - i think via babel : only allows 0..23 range for hour + total_seconds = o.total_seconds() + hours, remainder = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + rounded_seconds = round(seconds, 6) + int_seconds = int(seconds) + + if rounded_seconds == int_seconds: + seconds = int_seconds + else: + seconds = rounded_seconds + + return "{:01}:{:02}:{:02}".format(int(hours), int(minutes), seconds) + + +def parse_timedelta(s: str) -> datetime.timedelta: + # ref: https://stackoverflow.com/a/21074460/10309266 + if 'day' in s: + m = re.match(r"(?P[-\d]+) day[s]*, (?P\d+):(?P\d+):(?P\d[\.\d+]*)", s) + else: + m = re.match(r"(?P\d+):(?P\d+):(?P\d[\.\d+]*)", s) + + return datetime.timedelta(**{key: float(val) for key, val in m.groupdict().items()}) diff --git a/frappe/utils/formatters.py b/frappe/utils/formatters.py index 9436dea2c2..9916853caf 100644 --- a/frappe/utils/formatters.py +++ b/frappe/utils/formatters.py @@ -3,9 +3,11 @@ import frappe import datetime -from frappe.utils import formatdate, fmt_money, flt, cstr, cint, format_datetime, format_time, format_duration +from frappe.utils import formatdate, fmt_money, flt, cstr, cint, format_datetime, format_time, format_duration, format_timedelta from frappe.model.meta import get_field_currency, get_field_precision import re +from dateutil.parser import ParserError + def format_value(value, df=None, doc=None, currency=None, translated=False, format=None): '''Format value based on given fieldtype, document reference, currency reference. @@ -47,7 +49,10 @@ def format_value(value, df=None, doc=None, currency=None, translated=False, form return format_datetime(value) elif df.get("fieldtype")=="Time": - return format_time(value) + try: + return format_time(value) + except ParserError: + return format_timedelta(value) elif value==0 and df.get("fieldtype") in ("Int", "Float", "Currency", "Percent") and df.get("print_hide_if_no_value"): # this is required to show 0 as blank in table columns diff --git a/frappe/utils/response.py b/frappe/utils/response.py index f6ad91dbd2..a852c584c6 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import json @@ -16,7 +16,7 @@ from werkzeug.local import LocalProxy from werkzeug.wsgi import wrap_file from werkzeug.wrappers import Response from werkzeug.exceptions import NotFound, Forbidden -from frappe.utils import cint +from frappe.utils import cint, format_timedelta from urllib.parse import quote from frappe.core.doctype.access_log.access_log import make_access_log @@ -122,12 +122,14 @@ def make_logs(response = None): def json_handler(obj): """serialize non-serializable data for json""" - # serialize date - import collections.abc + from collections.abc import Iterable - if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime, datetime.time)): + if isinstance(obj, (datetime.date, datetime.datetime, datetime.time)): return str(obj) + elif isinstance(obj, datetime.timedelta): + return format_timedelta(obj) + elif isinstance(obj, decimal.Decimal): return float(obj) @@ -138,7 +140,7 @@ def json_handler(obj): doc = obj.as_dict(no_nulls=True) return doc - elif isinstance(obj, collections.abc.Iterable): + elif isinstance(obj, Iterable): return list(obj) elif type(obj)==type or isinstance(obj, Exception): diff --git a/frappe/website/doctype/web_form/templates/web_form.html b/frappe/website/doctype/web_form/templates/web_form.html index a8666b55e9..72cdf07c59 100644 --- a/frappe/website/doctype/web_form/templates/web_form.html +++ b/frappe/website/doctype/web_form/templates/web_form.html @@ -92,18 +92,12 @@ $(".file-size").each(function() { }); {{ include_script("controls.bundle.js") }} -{% if is_list %} -{# web form list #} - - +{% if is_list %} {{ include_script("dialog.bundle.js") }} {{ include_script("web_form.bundle.js") }} {{ include_script("bootstrap-4-web.bundle.js") }} -{% else %} -{# web form #} +{% else %} {{ include_script("dialog.bundle.js") }} - -