diff --git a/cypress/integration/control_link.js b/cypress/integration/control_link.js index 6d16769b37..bfa70ad338 100644 --- a/cypress/integration/control_link.js +++ b/cypress/integration/control_link.js @@ -58,6 +58,23 @@ context('Control Link', () => { cy.get('.frappe-control[data-fieldname=link] input').should('have.value', ''); }); + it("should be possible set empty value explicitly", () => { + get_dialog_with_link().as("dialog"); + + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); + + cy.get(".frappe-control[data-fieldname=link] input") + .type(" ", { delay: 100 }) + .blur(); + cy.wait("@validate_link"); + cy.get(".frappe-control[data-fieldname=link] input").should("have.value", ""); + cy.window() + .its("cur_dialog") + .then((dialog) => { + expect(dialog.get_value("link")).to.equal(''); + }); + }); + it('should route to form on arrow click', () => { get_dialog_with_link().as('dialog'); @@ -78,7 +95,7 @@ context('Control Link', () => { }); }); - it('should fetch valid value', () => { + it('should update dependant fields (via fetch_from)', () => { cy.get('@todos').then(todos => { cy.visit(`/app/todo/${todos[0]}`); cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); @@ -89,7 +106,67 @@ context('Control Link', () => { cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( 'contain', 'Administrator' ); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", "Administrator"); + + // invalid input + cy.get('@input').clear().type('invalid input', {delay: 100}).blur(); + cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( + 'contain', '' + ); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", null); + + // set valid value again + cy.get('@input').clear().type('Administrator', {delay: 100}).blur(); + cy.wait('@validate_link'); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", "Administrator"); + + // clear input + cy.get('@input').clear().blur(); + cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( + 'contain', '' + ); + + cy.window() + .its("cur_frm.doc.assigned_by") + .should("eq", ""); }); }); + it("should set default values", () => { + cy.insert_doc("Property Setter", { + "doctype_or_field": "DocField", + "doc_type": "ToDo", + "field_name": "assigned_by", + "property": "default", + "property_type": "Text", + "value": "Administrator" + }, true); + cy.reload(); + cy.new_form("ToDo"); + cy.fill_field("description", "new", "Text Editor"); + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); + cy.findByRole("button", {name: "Save"}).click(); + cy.wait("@save_form"); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", "Administrator" + ); + // if user clears default value explicitly, system should not reset default again + cy.get_field("assigned_by").clear().blur(); + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); + cy.findByRole("button", {name: "Save"}).click(); + cy.wait("@save_form"); + cy.get_field("assigned_by").should("have.value", ""); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", "" + ); + }); }); diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 758b3cde2b..4f273af21f 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -110,34 +110,6 @@ Cypress.Commands.add('get_doc', (doctype, name) => { }); }); -Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { - return cy - .window() - .its('frappe.csrf_token') - .then(csrf_token => { - return cy - .request({ - method: 'POST', - url: `/api/resource/${doctype}`, - body: args, - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - }, - failOnStatusCode: !ignore_duplicate - }) - .then(res => { - let status_codes = [200]; - if (ignore_duplicate) { - status_codes.push(409); - } - expect(res.status).to.be.oneOf(status_codes); - return res.body; - }); - }); -}); - Cypress.Commands.add('remove_doc', (doctype, name) => { return cy .window() diff --git a/frappe/__init__.py b/frappe/__init__.py index c6cbfead43..3558603454 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -143,6 +143,8 @@ lang = local("lang") # This if block is never executed when running the code. It is only used for # telling static code analyzer where to find dynamically defined attributes. if typing.TYPE_CHECKING: + from frappe.utils.redis_wrapper import RedisWrapper + from frappe.database.mariadb.database import MariaDBDatabase from frappe.database.postgres.database import PostgresDatabase from frappe.query_builder.builder import MariaDB, Postgres @@ -150,6 +152,7 @@ if typing.TYPE_CHECKING: db: typing.Union[MariaDBDatabase, PostgresDatabase] qb: typing.Union[MariaDB, Postgres] + # end: static analysis hack def init(site, sites_path=None, new_site=False): @@ -311,9 +314,8 @@ def destroy(): release_local(local) -# memcache redis_server = None -def cache(): +def cache() -> "RedisWrapper": """Returns redis connection.""" global redis_server if not redis_server: diff --git a/frappe/api.py b/frappe/api.py index b061761d10..e7f7bf5a04 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -94,7 +94,8 @@ def handle(): "data": doc.save().as_dict() }) - if doc.parenttype and doc.parent: + # check for child table doctype + if doc.get("parenttype"): frappe.get_doc(doc.parenttype, doc.parent).save() frappe.db.commit() diff --git a/frappe/app.py b/frappe/app.py index d73dd67983..609a8535d7 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -192,12 +192,7 @@ def make_form_dict(request): if not isinstance(args, dict): frappe.throw(_("Invalid request arguments")) - try: - frappe.local.form_dict = frappe._dict({ - k: v[0] if isinstance(v, (list, tuple)) else v for k, v in args.items() - }) - except IndexError: - frappe.local.form_dict = frappe._dict(args) + frappe.local.form_dict = frappe._dict(args) if "_" in frappe.local.form_dict: # _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict diff --git a/frappe/boot.py b/frappe/boot.py index f1fd0f6a3b..7f62d96cae 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -7,6 +7,7 @@ bootstrap client session import frappe import frappe.defaults import frappe.desk.desk_page +from frappe.desk.doctype.route_history.route_history import frequently_visited_links from frappe.desk.form.load import get_meta_bundle from frappe.utils.change_log import get_versions from frappe.translate import get_lang_dict @@ -15,7 +16,6 @@ from frappe.social.doctype.energy_point_settings.energy_point_settings import is from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points from frappe.model.base_document import get_controller -from frappe.social.doctype.post.post import frequently_visited_links from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings, get_app_logo from frappe.utils import get_time_zone, add_user_info diff --git a/frappe/client.py b/frappe/client.py index e835e7fee7..1898994afe 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -99,7 +99,6 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren if not filters: filters = None - if frappe.get_meta(doctype).issingle: value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug) else: @@ -129,7 +128,7 @@ def set_value(doctype, name, fieldname, value=None): :param fieldname: fieldname string or JSON / dict with key value pair :param value: value if fieldname is JSON / dict''' - if fieldname!="idx" and fieldname in frappe.model.default_fields: + if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): frappe.throw(_("Cannot edit standard fields")) if not value: @@ -142,14 +141,15 @@ def set_value(doctype, name, fieldname, value=None): else: values = {fieldname: value} - doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) - if doc and doc.parent and doc.parenttype: + # check for child table doctype + if not frappe.get_meta(doctype).istable: + doc = frappe.get_doc(doctype, name) + doc.update(values) + else: + doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) doc = frappe.get_doc(doc.parenttype, doc.parent) child = doc.getone({"doctype": doctype, "name": name}) child.update(values) - else: - doc = frappe.get_doc(doctype, name) - doc.update(values) doc.save() @@ -163,10 +163,10 @@ def insert(doc=None): if isinstance(doc, str): doc = json.loads(doc) - if doc.get("parent") and doc.get("parenttype"): + if doc.get("parenttype"): # inserting a child record - parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent")) - parent.append(doc.get("parentfield"), doc) + parent = frappe.get_doc(doc.parenttype, doc.parent) + parent.append(doc.parentfield, doc) parent.save() return parent.as_dict() else: @@ -187,10 +187,10 @@ def insert_many(docs=None): frappe.throw(_('Only 200 inserts allowed in one request')) for doc in docs: - if doc.get("parent") and doc.get("parenttype"): + if doc.get("parenttype"): # inserting a child record - parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent")) - parent.append(doc.get("parentfield"), doc) + parent = frappe.get_doc(doc.parenttype, doc.parent) + parent.append(doc.parentfield, doc) parent.save() out.append(parent.name) else: diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 41b607b192..e3379a43aa 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -623,6 +623,7 @@ def transform_database(context, table, engine, row_format, failfast): @click.command('run-tests') @click.option('--app', help="For App") @click.option('--doctype', help="For DocType") +@click.option('--case', help="Select particular TestCase") @click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt") @click.option('--test', multiple=True, help="Specific test") @click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests") @@ -636,7 +637,7 @@ def transform_database(context, table, engine, row_format, failfast): @pass_context def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False, coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None, - skip_test_records=False, skip_before_tests=False, failfast=False): + skip_test_records=False, skip_before_tests=False, failfast=False, case=None): with CodeCoverage(coverage, app): import frappe.test_runner @@ -658,7 +659,7 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests, force=context.force, profile=profile, junit_xml_output=junit_xml_output, - ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast) + ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast, case=case) if len(ret.failures) == 0 and len(ret.errors) == 0: ret = 0 diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 54ddbce2c4..46ef7bf5d2 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -1,29 +1,32 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import json -from email.utils import formataddr -from frappe.core.utils import get_parent_doc -from frappe.utils import (get_url, get_formatted_email, cint, list_to_str, - validate_email_address, split_emails, parse_addr, get_datetime) -from frappe.email.email_body import get_message_id +from typing import TYPE_CHECKING, Dict + +import frappe import frappe.email.smtp -import time from frappe import _ -from frappe.utils.background_jobs import enqueue +from frappe.email.email_body import get_message_id +from frappe.utils import (cint, get_datetime, get_formatted_email, + list_to_str, split_emails, validate_email_address) + +if TYPE_CHECKING: + from frappe.core.doctype.communication.communication import Communication + OUTGOING_EMAIL_ACCOUNT_MISSING = _(""" Unable to send mail because of a missing email account. Please setup default Email Account from Setup > Email > Email Account """) + @frappe.whitelist() def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent", sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False, print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, bcc=None, flags=None, read_receipt=None, print_letterhead=True, email_template=None, communication_type=None, - ignore_permissions=False): + ignore_permissions=False) -> Dict[str, str]: """Make a new communication. :param doctype: Reference DocType. @@ -56,7 +59,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = cc = list_to_str(cc) if isinstance(cc, list) else cc bcc = list_to_str(bcc) if isinstance(bcc, list) else bcc - comm = frappe.get_doc({ + comm: "Communication" = frappe.get_doc({ "doctype":"Communication", "subject": subject, "content": content, @@ -73,16 +76,13 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "message_id":get_message_id().strip(" <>"), "read_receipt":read_receipt, "has_attachment": 1 if attachments else 0, - "communication_type": communication_type + "communication_type": communication_type, }).insert(ignore_permissions=True) - comm.save(ignore_permissions=True) - - if isinstance(attachments, str): - attachments = json.loads(attachments) - # if not committed, delayed task doesn't find the communication if attachments: + if isinstance(attachments, str): + attachments = json.loads(attachments) add_attachments(comm.name, attachments) if cint(send_email): @@ -93,12 +93,13 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead) emails_not_sent_to = comm.exclude_emails_list(include_sender=send_me_a_copy) + return { "name": comm.name, - "emails_not_sent_to": ", ".join(emails_not_sent_to or []) + "emails_not_sent_to": ", ".join(emails_not_sent_to) } -def validate_email(doc): +def validate_email(doc: "Communication") -> None: """Validate Email Addresses of Recipients and CC""" if not (doc.communication_type=="Communication" and doc.communication_medium == "Email") or doc.flags.in_receive: return @@ -114,8 +115,6 @@ def validate_email(doc): for email in split_emails(doc.bcc): validate_email_address(email, throw=True) - # validate sender - def set_incoming_outgoing_accounts(doc): from frappe.email.doctype.email_account.email_account import EmailAccount incoming_email_account = EmailAccount.find_incoming( diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index b6d8070d00..dd9f58342e 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -1,3 +1,4 @@ +from typing import List import frappe from frappe import _ from frappe.core.utils import get_parent_doc @@ -194,14 +195,18 @@ class CommunicationEmailMixin: return _("Leave this conversation") return '' - def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False): + def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False) -> List: """List of mail id's excluded while sending mail. """ all_ids = self.get_all_email_addresses(exclude_displayname=True) - final_ids = self.mail_recipients(is_inbound_mail_communcation = is_inbound_mail_communcation) + \ - self.mail_bcc(is_inbound_mail_communcation = is_inbound_mail_communcation) + \ - self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender) - return set(all_ids) - set(final_ids) + + final_ids = ( + self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation) + + self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation) + + self.mail_cc(is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender) + ) + + return list(set(all_ids) - set(final_ids)) def get_assignees(self): """Get owners of the reference document. diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 216db53c72..dfc560a98a 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -44,6 +44,7 @@ frappe.ui.form.on('Data Import', { } frm.dashboard.show_progress(__('Import Progress'), percent, message); frm.page.set_indicator(__('In Progress'), 'orange'); + frm.trigger('update_primary_action'); // hide progress when complete if (data.current === data.total) { @@ -80,7 +81,10 @@ frappe.ui.form.on('Data Import', { frm.trigger('show_import_log'); frm.trigger('show_import_warnings'); frm.trigger('toggle_submit_after_import'); - frm.trigger('show_import_status'); + + if (frm.doc.status != 'Pending') + frm.trigger('show_import_status'); + frm.trigger('show_report_error_button'); if (frm.doc.status === 'Partial Success') { @@ -128,40 +132,49 @@ frappe.ui.form.on('Data Import', { }, show_import_status(frm) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); - let successful_records = import_log.filter(log => log.success); - let failed_records = import_log.filter(log => !log.success); - if (successful_records.length === 0) return; + frappe.call({ + 'method': 'frappe.core.doctype.data_import.data_import.get_import_status', + 'args': { + 'data_import_name': frm.doc.name + }, + 'callback': function(r) { + let successful_records = cint(r.message.success); + let failed_records = cint(r.message.failed); + let total_records = cint(r.message.total_records); - let message; - if (failed_records.length === 0) { - let message_args = [successful_records.length]; - if (frm.doc.import_type === 'Insert New Records') { - message = - successful_records.length > 1 - ? __('Successfully imported {0} records.', message_args) - : __('Successfully imported {0} record.', message_args); - } else { - message = - successful_records.length > 1 - ? __('Successfully updated {0} records.', message_args) - : __('Successfully updated {0} record.', message_args); + if (!total_records) return; + + let message; + if (failed_records === 0) { + let message_args = [successful_records]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records > 1 + ? __('Successfully imported {0} records.', message_args) + : __('Successfully imported {0} record.', message_args); + } else { + message = + successful_records > 1 + ? __('Successfully updated {0} records.', message_args) + : __('Successfully updated {0} record.', message_args); + } + } else { + let message_args = [successful_records, total_records]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records > 1 + ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) + : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + } else { + message = + successful_records > 1 + ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) + : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + } + } + frm.dashboard.set_headline(message); } - } else { - let message_args = [successful_records.length, import_log.length]; - if (frm.doc.import_type === 'Insert New Records') { - message = - successful_records.length > 1 - ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); - } else { - message = - successful_records.length > 1 - ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); - } - } - frm.dashboard.set_headline(message); + }); }, show_report_error_button(frm) { @@ -275,7 +288,7 @@ frappe.ui.form.on('Data Import', { }, show_import_preview(frm, preview_data) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); + let import_log = preview_data.import_log; if ( frm.import_preview && @@ -316,6 +329,15 @@ frappe.ui.form.on('Data Import', { ); }, + export_import_log(frm) { + open_url_post( + '/api/method/frappe.core.doctype.data_import.data_import.download_import_log', + { + data_import_name: frm.doc.name + } + ); + }, + show_import_warnings(frm, preview_data) { let columns = preview_data.columns; let warnings = JSON.parse(frm.doc.template_warnings || '[]'); @@ -391,92 +413,131 @@ frappe.ui.form.on('Data Import', { frm.trigger('show_import_log'); }, - show_import_log(frm) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); - let logs = import_log; - frm.toggle_display('import_log', false); - frm.toggle_display('import_log_section', logs.length > 0); + render_import_log(frm) { + frappe.call({ + 'method': 'frappe.client.get_list', + 'args': { + 'doctype': 'Data Import Log', + 'filters': { + 'data_import': frm.doc.name + }, + 'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'], + 'limit_page_length': 5000, + 'order_by': 'log_index' + }, + callback: function(r) { + let logs = r.message; - if (logs.length === 0) { - frm.get_field('import_log_preview').$wrapper.empty(); + if (logs.length === 0) return; + + frm.toggle_display('import_log_section', true); + + let rows = logs + .map(log => { + let html = ''; + if (log.success) { + if (frm.doc.import_type === 'Insert New Records') { + html = __('Successfully imported {0}', [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}` + ]); + } else { + html = __('Successfully updated {0}', [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}` + ]); + } + } else { + let messages = (JSON.parse(log.messages || '[]')) + .map(JSON.parse) + .map(m => { + let title = m.title ? `${m.title}` : ''; + let message = m.message ? `
${m.message}
` : ''; + return title + message; + }) + .join(''); + let id = frappe.dom.get_unique_id(); + html = `${messages} + +
+
+ 
${log.exception}
+
+
`; + } + let indicator_color = log.success ? 'green' : 'red'; + let title = log.success ? __('Success') : __('Failure'); + + if (frm.doc.show_failed_logs && log.success) { + return ''; + } + + return ` + ${JSON.parse(log.row_indexes).join(', ')} + +
${title}
+ + + ${html} + + `; + }) + .join(''); + + if (!rows && frm.doc.show_failed_logs) { + rows = ` + ${__('No failed logs')} + `; + } + + frm.get_field('import_log_preview').$wrapper.html(` + + + + + + + ${rows} +
${__('Row Number')}${__('Status')}${__('Message')}
+ `); + } + }); + }, + + show_import_log(frm) { + frm.toggle_display('import_log_section', false); + + if (frm.import_in_progress) { return; } - let rows = logs - .map(log => { - let html = ''; - if (log.success) { - if (frm.doc.import_type === 'Insert New Records') { - html = __('Successfully imported {0}', [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}` - ]); - } else { - html = __('Successfully updated {0}', [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}` - ]); - } + frappe.call({ + 'method': 'frappe.client.get_count', + 'args': { + 'doctype': 'Data Import Log', + 'filters': { + 'data_import': frm.doc.name + } + }, + 'callback': function(r) { + let count = r.message; + if (count < 5000) { + frm.trigger('render_import_log'); } else { - let messages = log.messages - .map(JSON.parse) - .map(m => { - let title = m.title ? `${m.title}` : ''; - let message = m.message ? `
${m.message}
` : ''; - return title + message; - }) - .join(''); - let id = frappe.dom.get_unique_id(); - html = `${messages} - -
-
- 
${log.exception}
-
-
`; + frm.toggle_display('import_log_section', false); + frm.add_custom_button(__('Export Import Log'), () => + frm.trigger('export_import_log') + ); } - let indicator_color = log.success ? 'green' : 'red'; - let title = log.success ? __('Success') : __('Failure'); - - if (frm.doc.show_failed_logs && log.success) { - return ''; - } - - return ` - ${log.row_indexes.join(', ')} - -
${title}
- - - ${html} - - `; - }) - .join(''); - - if (!rows && frm.doc.show_failed_logs) { - rows = ` - ${__('No failed logs')} - `; - } - - frm.get_field('import_log_preview').$wrapper.html(` - - - - - - - ${rows} -
${__('Row Number')}${__('Status')}${__('Message')}
- `); + } + }); }, }); diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index fe6fb90481..9e948dac8c 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -1,194 +1,197 @@ { - "actions": [], - "autoname": "format:{reference_doctype} Import on {creation}", - "beta": 1, - "creation": "2019-08-04 14:16:08.318714", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "import_type", - "download_template", - "import_file", - "html_5", - "google_sheets_url", - "refresh_google_sheet", - "column_break_5", - "status", - "submit_after_import", - "mute_emails", - "template_options", - "import_warnings_section", - "template_warnings", - "import_warnings", - "section_import_preview", - "import_preview", - "import_log_section", - "import_log", - "show_failed_logs", - "import_log_preview" - ], - "fields": [ - { - "fieldname": "reference_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1, - "set_only_once": 1 - }, - { - "fieldname": "import_type", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Import Type", - "options": "\nInsert New Records\nUpdate Existing Records", - "reqd": 1, - "set_only_once": 1 - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "import_file", - "fieldtype": "Attach", - "in_list_view": 1, - "label": "Import File", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "fieldname": "import_preview", - "fieldtype": "HTML", - "label": "Import Preview" - }, - { - "fieldname": "section_import_preview", - "fieldtype": "Section Break", - "label": "Preview" - }, - { - "fieldname": "column_break_5", - "fieldtype": "Column Break" - }, - { - "fieldname": "template_options", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Options", - "options": "JSON", - "read_only": 1 - }, - { - "fieldname": "import_log", - "fieldtype": "Code", - "label": "Import Log", - "options": "JSON" - }, - { - "fieldname": "import_log_section", - "fieldtype": "Section Break", - "label": "Import Log" - }, - { - "fieldname": "import_log_preview", - "fieldtype": "HTML", - "label": "Import Log Preview" - }, - { - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 1, - "label": "Status", - "options": "Pending\nSuccess\nPartial Success\nError", - "read_only": 1 - }, - { - "fieldname": "template_warnings", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Warnings", - "options": "JSON" - }, - { - "default": "0", - "fieldname": "submit_after_import", - "fieldtype": "Check", - "label": "Submit After Import", - "set_only_once": 1 - }, - { - "fieldname": "import_warnings_section", - "fieldtype": "Section Break", - "label": "Import File Errors and Warnings" - }, - { - "fieldname": "import_warnings", - "fieldtype": "HTML", - "label": "Import Warnings" - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "download_template", - "fieldtype": "Button", - "label": "Download Template" - }, - { - "default": "1", - "fieldname": "mute_emails", - "fieldtype": "Check", - "label": "Don't Send Emails", - "set_only_once": 1 - }, - { - "default": "0", - "fieldname": "show_failed_logs", - "fieldtype": "Check", - "label": "Show Failed Logs" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file", - "fieldname": "html_5", - "fieldtype": "HTML", - "options": "
Or
" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file\n", - "description": "Must be a publicly accessible Google Sheets URL", - "fieldname": "google_sheets_url", - "fieldtype": "Data", - "label": "Import from Google Sheets", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", - "fieldname": "refresh_google_sheet", - "fieldtype": "Button", - "label": "Refresh Google Sheet" - } - ], - "hide_toolbar": 1, - "links": [], - "modified": "2021-04-11 01:50:42.074623", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 + "actions": [], + "autoname": "format:{reference_doctype} Import on {creation}", + "beta": 1, + "creation": "2019-08-04 14:16:08.318714", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "import_type", + "download_template", + "import_file", + "payload_count", + "html_5", + "google_sheets_url", + "refresh_google_sheet", + "column_break_5", + "status", + "submit_after_import", + "mute_emails", + "template_options", + "import_warnings_section", + "template_warnings", + "import_warnings", + "section_import_preview", + "import_preview", + "import_log_section", + "show_failed_logs", + "import_log_preview" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "import_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Import Type", + "options": "\nInsert New Records\nUpdate Existing Records", + "reqd": 1, + "set_only_once": 1 + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "import_file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Import File", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "fieldname": "import_preview", + "fieldtype": "HTML", + "label": "Import Preview" + }, + { + "fieldname": "section_import_preview", + "fieldtype": "Section Break", + "label": "Preview" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "template_options", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Options", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "import_log_section", + "fieldtype": "Section Break", + "label": "Import Log" + }, + { + "fieldname": "import_log_preview", + "fieldtype": "HTML", + "label": "Import Log Preview" + }, + { + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "label": "Status", + "options": "Pending\nSuccess\nPartial Success\nError", + "read_only": 1 + }, + { + "fieldname": "template_warnings", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Warnings", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "label": "Submit After Import", + "set_only_once": 1 + }, + { + "fieldname": "import_warnings_section", + "fieldtype": "Section Break", + "label": "Import File Errors and Warnings" + }, + { + "fieldname": "import_warnings", + "fieldtype": "HTML", + "label": "Import Warnings" + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" + }, + { + "default": "1", + "fieldname": "mute_emails", + "fieldtype": "Check", + "label": "Don't Send Emails", + "set_only_once": 1 + }, + { + "default": "0", + "fieldname": "show_failed_logs", + "fieldtype": "Check", + "label": "Show Failed Logs" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file", + "fieldname": "html_5", + "fieldtype": "HTML", + "options": "
Or
" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file\n", + "description": "Must be a publicly accessible Google Sheets URL", + "fieldname": "google_sheets_url", + "fieldtype": "Data", + "label": "Import from Google Sheets", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", + "fieldname": "refresh_google_sheet", + "fieldtype": "Button", + "label": "Refresh Google Sheet" + }, + { + "fieldname": "payload_count", + "fieldtype": "Int", + "hidden": 1, + "label": "Payload Count", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "links": [], + "modified": "2022-02-01 20:08:37.624914", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 5935ddc4ba..5972e79b4d 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -27,6 +27,7 @@ class DataImport(Document): self.validate_import_file() self.validate_google_sheets_url() + self.set_payload_count() def validate_import_file(self): if self.import_file: @@ -38,6 +39,12 @@ class DataImport(Document): return validate_google_sheets_url(self.google_sheets_url) + def set_payload_count(self): + if self.import_file: + i = self.get_importer() + payloads = i.import_file.get_payloads_for_import() + self.payload_count = len(payloads) + @frappe.whitelist() def get_preview_from_template(self, import_file=None, google_sheets_url=None): if import_file: @@ -67,7 +74,7 @@ class DataImport(Document): enqueue( start_import, queue="default", - timeout=6000, + timeout=10000, event="data_import", job_name=self.name, data_import=self.name, @@ -80,6 +87,9 @@ class DataImport(Document): def export_errored_rows(self): return self.get_importer().export_errored_rows() + def download_import_log(self): + return self.get_importer().export_import_log() + def get_importer(self): return Importer(self.reference_doctype, data_import=self) @@ -90,7 +100,6 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N import_file, google_sheets_url ) - @frappe.whitelist() def form_start_import(data_import): return frappe.get_doc("Data Import", data_import).start_import() @@ -145,6 +154,30 @@ def download_errored_template(data_import_name): data_import = frappe.get_doc("Data Import", data_import_name) data_import.export_errored_rows() +@frappe.whitelist() +def download_import_log(data_import_name): + data_import = frappe.get_doc("Data Import", data_import_name) + data_import.download_import_log() + +@frappe.whitelist() +def get_import_status(data_import_name): + import_status = {} + + logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'], + filters={'data_import': data_import_name}, + group_by='success') + + total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count') + + for log in logs: + if log.get('success'): + import_status['success'] = log.get('count') + else: + import_status['failed'] = log.get('count') + + import_status['total_records'] = total_payload_count + + return import_status def import_file( doctype, file_path, import_type, submit_after_import=False, console=False diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 0eb05aa354..6ab750ba25 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -24,12 +24,14 @@ frappe.listview_settings['Data Import'] = { 'Error': 'red' }; let status = doc.status; + if (imports_in_progress.includes(doc.name)) { status = 'In Progress'; } if (status == 'Pending') { status = 'Not Started'; } + return [__(status), colors[status], 'status,=,' + doc.status]; }, formatters: { diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index b9b2050763..f085709945 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -47,7 +47,13 @@ class Importer: ) def get_data_for_import_preview(self): - return self.import_file.get_data_for_import_preview() + out = self.import_file.get_data_for_import_preview() + + out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index", limit=10) + + return out def before_import(self): # set user lang for translations @@ -58,7 +64,6 @@ class Importer: frappe.flags.in_import = True frappe.flags.mute_emails = self.data_import.mute_emails - self.data_import.db_set("status", "Pending") self.data_import.db_set("template_warnings", "") def import_data(self): @@ -79,20 +84,25 @@ class Importer: return # setup import log - if self.data_import.import_log: - import_log = frappe.parse_json(self.data_import.import_log) - else: - import_log = [] + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] - # remove previous failures from import log - import_log = [log for log in import_log if log.get("success")] + log_index = 0 + + # Do not remove rows in case of retry after an error or pending data import + if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count: + # remove previous failures from import log only in case of retry after partial success + import_log = [log for log in import_log if log.get("success")] # get successfully imported rows imported_rows = [] for log in import_log: log = frappe._dict(log) - if log.success: - imported_rows += log.row_indexes + if log.success or len(import_log) < self.data_import.payload_count: + imported_rows += json.loads(log.row_indexes) + + log_index = log.log_index # start import total_payload_count = len(payloads) @@ -146,25 +156,41 @@ class Importer: }, ) - import_log.append( - frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) - ) + create_import_log(self.data_import.name, log_index, { + 'success': True, + 'docname': doc.name, + 'row_indexes': row_indexes + }) + + log_index += 1 + + if not self.data_import.status == "Partial Success": + self.data_import.db_set("status", "Partial Success") + # commit after every successful import frappe.db.commit() except Exception: - import_log.append( - frappe._dict( - success=False, - exception=frappe.get_traceback(), - messages=frappe.local.message_log, - row_indexes=row_indexes, - ) - ) + messages = frappe.local.message_log frappe.clear_messages() + # rollback if exception frappe.db.rollback() + create_import_log(self.data_import.name, log_index, { + 'success': False, + 'exception': frappe.get_traceback(), + 'messages': messages, + 'row_indexes': row_indexes + }) + + log_index += 1 + + # Logs are db inserted directly so will have to be fetched again + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + # set status failures = [log for log in import_log if not log.get("success")] if len(failures) == total_payload_count: @@ -178,7 +204,6 @@ class Importer: self.print_import_log(import_log) else: self.data_import.db_set("status", status) - self.data_import.db_set("import_log", json.dumps(import_log)) self.after_import() @@ -248,11 +273,14 @@ class Importer: if not self.data_import: return - import_log = frappe.parse_json(self.data_import.import_log or "[]") + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + failures = [log for log in import_log if not log.get("success")] row_indexes = [] for f in failures: - row_indexes.extend(f.get("row_indexes", [])) + row_indexes.extend(json.loads(f.get("row_indexes", []))) # de duplicate row_indexes = list(set(row_indexes)) @@ -264,6 +292,30 @@ class Importer: build_csv_response(rows, _(self.doctype)) + def export_import_log(self): + from frappe.utils.csvutils import build_csv_response + + if not self.data_import: + return + + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": self.data_import.name}, + order_by="log_index") + + header_row = ["Row Numbers", "Status", "Message", "Exception"] + + rows = [header_row] + + for log in import_log: + row_number = json.loads(log.get("row_indexes"))[0] + status = "Success" if log.get('success') else "Failure" + message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \ + log.get("messages") + exception = frappe.utils.cstr(log.get("exception", '')) + rows += [[row_number, status, message, exception]] + + build_csv_response(rows, self.doctype) + def print_import_log(self, import_log): failed_records = [log for log in import_log if not log.success] successful_records = [log for log in import_log if log.success] @@ -566,7 +618,7 @@ class Row: ) # remove standard fields and __islocal - for key in frappe.model.default_fields + ("__islocal",): + for key in frappe.model.default_fields + frappe.model.child_table_fields + ("__islocal",): doc.pop(key, None) for col, value in zip(columns, values): @@ -1172,3 +1224,17 @@ def df_as_json(df): def get_select_options(df): return [d for d in (df.options or "").split("\n") if d] + +def create_import_log(data_import, log_index, log_details): + frappe.get_doc({ + 'doctype': 'Data Import Log', + 'log_index': log_index, + 'success': log_details.get('success'), + 'data_import': data_import, + 'row_indexes': json.dumps(log_details.get('row_indexes')), + 'docname': log_details.get('docname'), + 'messages': json.dumps(log_details.get('messages', '[]')), + 'exception': log_details.get('exception') + }).db_insert() + + diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index e1bc0e7ca5..11077ca58b 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -4,6 +4,7 @@ import unittest import frappe from frappe.core.doctype.data_import.importer import Importer +from frappe.tests.test_query_builder import db_type_is, run_only_if from frappe.utils import getdate, format_duration doctype_name = 'DocType for Import' @@ -54,21 +55,27 @@ class TestImporter(unittest.TestCase): self.assertEqual(len(preview.data), 4) self.assertEqual(len(preview.columns), 16) + # ignored on postgres because myisam doesn't exist on pg + @run_only_if(db_type_is.MARIADB) def test_data_import_without_mandatory_values(self): import_file = get_import_file('sample_import_file_without_mandatory') data_import = self.get_importer(doctype_name, import_file) frappe.local.message_log = [] data_import.start_import() data_import.reload() - import_log = frappe.parse_json(data_import.import_log) - self.assertEqual(import_log[0]['row_indexes'], [2,3]) - expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error) - expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error) - self.assertEqual(import_log[1]['row_indexes'], [4]) - self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required") + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": data_import.name}, + order_by="log_index") + + self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3]) + expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error) + expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error) + + self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4]) + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required") def test_data_import_update(self): existing_doc = frappe.get_doc( diff --git a/frappe/social/doctype/post/__init__.py b/frappe/core/doctype/data_import_log/__init__.py similarity index 100% rename from frappe/social/doctype/post/__init__.py rename to frappe/core/doctype/data_import_log/__init__.py diff --git a/frappe/core/doctype/data_import_log/data_import_log.js b/frappe/core/doctype/data_import_log/data_import_log.js new file mode 100644 index 0000000000..c376edeec9 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2021, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Data Import Log', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/data_import_log/data_import_log.json b/frappe/core/doctype/data_import_log/data_import_log.json new file mode 100644 index 0000000000..b1d991f099 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.json @@ -0,0 +1,84 @@ +{ + "actions": [], + "creation": "2021-12-25 16:12:20.205889", + "doctype": "DocType", + "editable_grid": 1, + "engine": "MyISAM", + "field_order": [ + "data_import", + "row_indexes", + "success", + "docname", + "messages", + "exception", + "log_index" + ], + "fields": [ + { + "fieldname": "data_import", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Data Import", + "options": "Data Import" + }, + { + "fieldname": "docname", + "fieldtype": "Data", + "label": "Reference Name" + }, + { + "fieldname": "exception", + "fieldtype": "Text", + "label": "Exception" + }, + { + "fieldname": "row_indexes", + "fieldtype": "Code", + "label": "Row Indexes", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "success", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Success" + }, + { + "fieldname": "log_index", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Log Index" + }, + { + "fieldname": "messages", + "fieldtype": "Code", + "label": "Messages", + "options": "JSON" + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2021-12-29 11:19:19.646076", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import Log", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC" +} \ No newline at end of file diff --git a/frappe/core/doctype/data_import_log/data_import_log.py b/frappe/core/doctype/data_import_log/data_import_log.py new file mode 100644 index 0000000000..a71aefa8bc --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class DataImportLog(Document): + pass diff --git a/frappe/core/doctype/data_import_log/test_data_import_log.py b/frappe/core/doctype/data_import_log/test_data_import_log.py new file mode 100644 index 0000000000..244404936e --- /dev/null +++ b/frappe/core/doctype/data_import_log/test_data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and Contributors +# See license.txt + +# import frappe +import unittest + +class TestDataImportLog(unittest.TestCase): + pass diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3754288145..d259367a16 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -10,7 +10,9 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache import frappe from frappe import _ from frappe.utils import now, cint -from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options +from frappe.model import ( + no_value_fields, default_fields, table_fields, data_field_options, child_table_fields +) from frappe.model.document import Document from frappe.model.base_document import get_controller from frappe.custom.doctype.property_setter.property_setter import make_property_setter @@ -74,6 +76,7 @@ class DocType(Document): self.make_amendable() self.make_repeatable() self.validate_nestedset() + self.validate_child_table() self.validate_website() self.ensure_minimum_max_attachment_limit() validate_links_table_fieldnames(self) @@ -689,6 +692,22 @@ class DocType(Document): }) self.nsm_parent_field = parent_field_name + def validate_child_table(self): + if not self.get("istable") or self.is_new(): + # if the doctype is not a child table then return + # if the doctype is a new doctype and also a child table then + # don't move forward as it will be handled via schema + return + + self.add_child_table_fields() + + def add_child_table_fields(self): + from frappe.database.schema import add_column + + add_column(self.name, "parent", "Data") + add_column(self.name, "parenttype", "Data") + add_column(self.name, "parentfield", "Data") + def get_max_idx(self): """Returns the highest `idx`""" max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", @@ -699,6 +718,13 @@ class DocType(Document): if not name: name = self.name + # a Doctype name is the tablename created in database + # `tab` the length of tablename is limited to 64 characters + max_length = frappe.db.MAX_COLUMN_LENGTH - 3 + if len(name) > max_length: + # length(tab + ) should be equal to 64 characters hence doctype should be 61 characters + frappe.throw(_("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError) + flags = {"flags": re.ASCII} # a DocType name should not start or end with an empty space @@ -1009,7 +1035,7 @@ def validate_fields(meta): sort_fields = [d.split()[0] for d in meta.sort_field.split(',')] for fieldname in sort_fields: - if not fieldname in fieldname_list + list(default_fields): + if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)): frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname), InvalidFieldNameError) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 12c227464d..50882f51bd 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -23,6 +23,7 @@ class TestDocType(unittest.TestCase): self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert) self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert) self.assertRaises(frappe.NameError, new_doctype("Some (DocType)").insert) + self.assertRaises(frappe.NameError, new_doctype("Some Doctype with a name whose length is more than 61 characters").insert) for name in ("Some DocType", "Some_DocType"): if frappe.db.exists("DocType", name): frappe.delete_doc("DocType", name) @@ -353,7 +354,6 @@ class TestDocType(unittest.TestCase): dump_docs = json.dumps(docs.get('docs')) cancel_all_linked_docs(dump_docs) data_link_doc.cancel() - data_doc.name = '{}-CANC-0'.format(data_doc.name) data_doc.load_from_db() self.assertEqual(data_link_doc.docstatus, 2) self.assertEqual(data_doc.docstatus, 2) @@ -377,7 +377,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - link_doc.insert(ignore_if_duplicate=True) + link_doc.insert() #create first parent doctype test_doc_1 = new_doctype('Test Doctype 1') @@ -392,7 +392,7 @@ class TestDocType(unittest.TestCase): for data in test_doc_1.get('permissions'): data.submit = 1 data.cancel = 1 - test_doc_1.insert(ignore_if_duplicate=True) + test_doc_1.insert() #crete second parent doctype doc = new_doctype('Test Doctype 2') @@ -407,7 +407,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - doc.insert(ignore_if_duplicate=True) + doc.insert() # create doctype data data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') @@ -438,7 +438,6 @@ class TestDocType(unittest.TestCase): # checking that doc for Test Doctype 2 is not canceled self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) - data_doc_2.name = '{}-CANC-0'.format(data_doc_2.name) data_doc.load_from_db() data_doc_2.load_from_db() self.assertEqual(data_link_doc_1.docstatus, 2) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index adf10b9a03..2808a2710b 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE """ @@ -7,7 +7,6 @@ record of files naming for same name files: file.gif, file-1.gif, file-2.gif etc """ -import base64 import hashlib import imghdr import io @@ -17,9 +16,10 @@ import os import re import shutil import zipfile +from typing import TYPE_CHECKING, Tuple import requests -import requests.exceptions +from requests.exceptions import HTTPError, SSLError from PIL import Image, ImageFile, ImageOps from io import BytesIO from urllib.parse import quote, unquote @@ -31,6 +31,11 @@ from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, g from frappe.utils.image import strip_exif_data, optimize_image from frappe.utils.file_manager import safe_b64decode +if TYPE_CHECKING: + from PIL.ImageFile import ImageFile + from requests.models import Response + + class MaxFileSizeReachedError(frappe.ValidationError): pass @@ -276,7 +281,7 @@ class File(Document): image, filename, extn = get_local_image(self.file_url) else: image, filename, extn = get_web_image(self.file_url) - except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError): + except (HTTPError, SSLError, IOError, TypeError): return size = width, height @@ -572,12 +577,10 @@ class File(Document): @staticmethod def zip_files(files): - from six import string_types - zip_file = io.BytesIO() zf = zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) for _file in files: - if isinstance(_file, string_types): + if isinstance(_file, str): _file = frappe.get_doc("File", _file) if not isinstance(_file, File): continue @@ -650,9 +653,17 @@ def setup_folder_path(filename, new_parent): from frappe.model.rename_doc import rename_doc rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True) -def get_extension(filename, extn, content): +def get_extension(filename, extn, content: bytes = None, response: "Response" = None) -> str: mimetype = None + if response: + content_type = response.headers.get("Content-Type") + + if content_type: + _extn = mimetypes.guess_extension(content_type) + if _extn: + return _extn[1:] + if extn: # remove '?' char and parameters from extn if present if '?' in extn: @@ -695,14 +706,14 @@ def get_local_image(file_url): return image, filename, extn -def get_web_image(file_url): +def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]: # download file_url = frappe.utils.get_url(file_url) r = requests.get(file_url, stream=True) try: r.raise_for_status() - except requests.exceptions.HTTPError as e: - if "404" in e.args[0]: + except HTTPError: + if r.status_code == 404: frappe.msgprint(_("File '{0}' not found").format(file_url)) else: frappe.msgprint(_("Unable to read file format for {0}").format(file_url)) @@ -721,7 +732,10 @@ def get_web_image(file_url): filename = get_random_filename() extn = None - extn = get_extension(filename, extn, r.content) + extn = get_extension(filename, extn, response=r) + if extn == "bin": + extn = get_extension(filename, extn, content=r.content) or "png" + filename = "/files/" + strip(unquote(filename)) return image, filename, extn @@ -864,8 +878,9 @@ def extract_images_from_html(doc, content, is_private=False): else: filename = get_random_filename(content_type=mtype) - doctype = doc.parenttype if doc.parent else doc.doctype - name = doc.parent or doc.name + # attaching a file to a child table doc, attaches it to the parent doc + doctype = doc.parenttype if doc.get("parent") else doc.doctype + name = doc.get("parent") or doc.name _file = frappe.get_doc({ "doctype": "File", diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2c1042e104..ba83dfca19 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import base64 import json import frappe import os import unittest + from frappe import _ -from frappe.core.doctype.file.file import get_attached_images, move_file, get_files_in_folder, unzip_file +from frappe.core.doctype.file.file import File, get_attached_images, move_file, get_files_in_folder, unzip_file from frappe.utils import get_files_path -# test_records = frappe.get_test_records('File') test_content1 = 'Hello' test_content2 = 'Hello World' @@ -24,8 +23,6 @@ def make_test_doc(): class TestSimpleFile(unittest.TestCase): - - def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = test_content1 @@ -38,21 +35,13 @@ class TestSimpleFile(unittest.TestCase): _file.save() self.saved_file_url = _file.file_url - def test_save(self): _file = frappe.get_doc("File", {"file_url": self.saved_file_url}) content = _file.get_content() self.assertEqual(content, self.test_content) - def tearDown(self): - # File gets deleted on rollback, so blank - pass - - class TestBase64File(unittest.TestCase): - - def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = base64.b64encode(test_content1.encode('utf-8')) @@ -66,18 +55,12 @@ class TestBase64File(unittest.TestCase): _file.save() self.saved_file_url = _file.file_url - def test_saved_content(self): _file = frappe.get_doc("File", {"file_url": self.saved_file_url}) content = _file.get_content() self.assertEqual(content, test_content1) - def tearDown(self): - # File gets deleted on rollback, so blank - pass - - class TestSameFileName(unittest.TestCase): def test_saved_content(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() @@ -130,8 +113,6 @@ class TestSameFileName(unittest.TestCase): class TestSameContent(unittest.TestCase): - - def setUp(self): self.attached_to_doctype1, self.attached_to_docname1 = make_test_doc() self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc() @@ -186,10 +167,6 @@ class TestSameContent(unittest.TestCase): limit_property.delete() frappe.clear_cache(doctype='ToDo') - def tearDown(self): - # File gets deleted on rollback, so blank - pass - class TestFile(unittest.TestCase): def setUp(self): @@ -398,7 +375,7 @@ class TestFile(unittest.TestCase): def test_make_thumbnail(self): # test web image - test_file = frappe.get_doc({ + test_file: File = frappe.get_doc({ "doctype": "File", "file_name": 'logo', "file_url": frappe.utils.get_url('/_test/assets/image.jpg'), @@ -407,6 +384,16 @@ class TestFile(unittest.TestCase): test_file.make_thumbnail() self.assertEquals(test_file.thumbnail_url, '/files/image_small.jpg') + # test web image without extension + test_file = frappe.get_doc({ + "doctype": "File", + "file_name": 'logo', + "file_url": frappe.utils.get_url('/_test/assets/image'), + }).insert(ignore_permissions=True) + + test_file.make_thumbnail() + self.assertTrue(test_file.thumbnail_url.endswith("_small.jpeg")) + # test local image test_file.db_set('thumbnail_url', None) test_file.reload() diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 266017dd71..9cb40dffd4 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -61,7 +61,7 @@ class Report(Document): delete_permanently=True) def get_columns(self): - return [d.as_dict(no_default_fields = True) for d in self.columns] + return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns] @frappe.whitelist() def set_doctype_roles(self): diff --git a/frappe/core/doctype/server_script/server_script_utils.py b/frappe/core/doctype/server_script/server_script_utils.py index d8c945fb6d..b5f3ba7168 100644 --- a/frappe/core/doctype/server_script/server_script_utils.py +++ b/frappe/core/doctype/server_script/server_script_utils.py @@ -34,19 +34,7 @@ def run_server_script_for_doc_event(doc, event): if scripts: # run all scripts for this doctype + event for script_name in scripts: - try: - frappe.get_doc('Server Script', script_name).execute_doc(doc) - except Exception as e: - message = frappe._('Error executing Server Script {0}. Open Browser Console to see traceback.').format( - frappe.utils.get_link_to_form('Server Script', script_name) - ) - exception = type(e) - if getattr(frappe, 'request', None): - # all exceptions throw 500 which is internal server error - # however server script error is a user error - # so we should throw 417 which is expectation failed - exception.http_status_code = 417 - frappe.throw(title=frappe._('Server Script Error'), msg=message, exc=exception) + frappe.get_doc('Server Script', script_name).execute_doc(doc) def get_server_script_map(): # fetch cached server script methods diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index bc92061f42..d9381bcd16 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -139,3 +139,42 @@ class TestServerScript(unittest.TestCase): server_script.disabled = 1 server_script.save() + + def test_restricted_qb(self): + todo = frappe.get_doc(doctype="ToDo", description="QbScriptTestNote") + todo.insert() + + script = frappe.get_doc( + doctype='Server Script', + name='test_qb_restrictions', + script_type = 'API', + api_method = 'test_qb_restrictions', + allow_guest = 1, + # whitelisted update + script = f''' +frappe.db.set_value("ToDo", "{todo.name}", "description", "safe") +''' + ) + script.insert() + script.execute_method() + + todo.reload() + self.assertEqual(todo.description, "safe") + + # unsafe update + script.script = f""" +todo = frappe.qb.DocType("ToDo") +frappe.qb.update(todo).set(todo.description, "unsafe").where(todo.name == "{todo.name}").run() +""" + script.save() + self.assertRaises(frappe.PermissionError, script.execute_method) + todo.reload() + self.assertEqual(todo.description, "safe") + + # safe select + script.script = f""" +todo = frappe.qb.DocType("ToDo") +frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run() +""" + script.save() + script.execute_method() diff --git a/frappe/core/doctype/user/test_user.py b/frappe/core/doctype/user/test_user.py index d1291acfc4..2d3da791ff 100644 --- a/frappe/core/doctype/user/test_user.py +++ b/frappe/core/doctype/user/test_user.py @@ -355,7 +355,10 @@ class TestUser(unittest.TestCase): test_user.reload() self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/") update_password(old_password, old_password=new_password) - self.assertEqual(json.loads(frappe.message_log[0]), {"message": "Password reset instructions have been sent to your email"}) + self.assertEqual( + json.loads(frappe.message_log[0]).get("message"), + "Password reset instructions have been sent to your email" + ) sendmail.assert_called_once() self.assertEqual(sendmail.call_args[1]["recipients"], "test2@example.com") diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index f1ccc25c6e..d08755f9a8 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -756,7 +756,7 @@ def verify_password(password): @frappe.whitelist(allow_guest=True) def sign_up(email, full_name, redirect_to): if is_signup_disabled(): - frappe.throw(_('Sign Up is disabled'), title='Not Allowed') + frappe.throw(_("Sign Up is disabled"), title=_("Not Allowed")) user = frappe.db.get("User", {"email": email}) if user: @@ -810,8 +810,10 @@ def reset_password(user): user.validate_reset_password() user.reset_password(send_email=True) - return frappe.msgprint(_("Password reset instructions have been sent to your email")) - + return frappe.msgprint( + msg=_("Password reset instructions have been sent to your email"), + title=_("Password Email Sent") + ) except frappe.DoesNotExistError: frappe.local.response['http_status_code'] = 400 frappe.clear_messages() diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index cf905c2ce2..d4a9d68fd5 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -3,6 +3,7 @@ from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable from frappe.permissions import has_user_permission from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.website.doctype.blog_post.test_blog_post import make_test_blog import frappe import unittest @@ -31,6 +32,18 @@ class TestUserPermission(unittest.TestCase): param = get_params(user, 'User', perm_user.name, is_default=1) self.assertRaises(frappe.ValidationError, add_user_permissions, param) + def test_default_user_permission_corectness(self): + user = create_user('test_default_corectness_permission_1@example.com') + param = get_params(user, 'User', user.name, is_default=1, hide_descendants= 1) + add_user_permissions(param) + #create a duplicate entry with default + perm_user = create_user('test_default_corectness2@example.com') + test_blog = make_test_blog() + param = get_params(perm_user, 'Blog Post', test_blog.name, is_default=1, hide_descendants= 1) + add_user_permissions(param) + frappe.db.delete('User Permission', filters={'for_value': test_blog.name}) + frappe.delete_doc('Blog Post', test_blog.name) + def test_default_user_permission(self): frappe.set_user('Administrator') user = create_user('test_user_perm1@example.com', 'Website Manager') diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 1366ace115..fb658481b2 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -48,7 +48,6 @@ class UserPermission(Document): }, or_filters={ 'applicable_for': cstr(self.applicable_for), 'apply_to_all_doctypes': 1, - 'hide_descendants': cstr(self.hide_descendants) }, limit=1) if overlap_exists: ref_link = frappe.get_desk_link(self.doctype, overlap_exists[0].name) diff --git a/frappe/core/doctype/user_type/test_user_type.py b/frappe/core/doctype/user_type/test_user_type.py index 7080e1830b..6807f8fc9e 100644 --- a/frappe/core/doctype/user_type/test_user_type.py +++ b/frappe/core/doctype/user_type/test_user_type.py @@ -1,8 +1,68 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE -# import frappe +import frappe import unittest +from frappe.installer import update_site_config + class TestUserType(unittest.TestCase): - pass + def setUp(self): + create_role() + + def test_add_select_perm_doctypes(self): + user_type = create_user_type('Test User Type') + + # select perms added for all link fields + doc = frappe.get_meta('Contact') + link_fields = doc.get_link_fields() + select_doctypes = frappe.get_all('User Select Document Type', {'parent': user_type.name}, pluck='document_type') + + for entry in link_fields: + self.assertTrue(entry.options in select_doctypes) + + # select perms added for all child table link fields + link_fields = [] + for child_table in doc.get_table_fields(): + child_doc = frappe.get_meta(child_table.options) + link_fields.extend(child_doc.get_link_fields()) + + for entry in link_fields: + self.assertTrue(entry.options in select_doctypes) + + def tearDown(self): + frappe.db.rollback() + + +def create_user_type(user_type): + if frappe.db.exists('User Type', user_type): + frappe.delete_doc('User Type', user_type) + + user_type_limit = {frappe.scrub(user_type): 1} + update_site_config('user_type_doctype_limit', user_type_limit) + + doc = frappe.get_doc({ + 'doctype': 'User Type', + 'name': user_type, + 'role': '_Test User Type', + 'user_id_field': 'user', + 'apply_user_permission_on': 'User' + }) + + doc.append('user_doctypes', { + 'document_type': 'Contact', + 'read': 1, + 'write': 1 + }) + + return doc.insert() + + +def create_role(): + if not frappe.db.exists('Role', '_Test User Type'): + frappe.get_doc({ + 'doctype': 'Role', + 'role_name': '_Test User Type', + 'desk_access': 1, + 'is_custom': 1 + }).insert() \ No newline at end of file diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 626ab772b8..c0dfd2e597 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters ['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]] doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters, - order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1) + order_by='`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1) custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)], ['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']] diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index be3e723af6..5f41f217f0 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -39,43 +39,3 @@ def get_todays_events(as_list=False): today = nowdate() events = get_events(today, today) return events if as_list else len(events) - -def get_unseen_likes(): - """Returns count of unseen likes""" - - comment_doctype = DocType("Comment") - return frappe.db.count(comment_doctype, - filters=( - (comment_doctype.comment_type == "Like") - & (comment_doctype.modified >= Now() - Interval(years=1)) - & (comment_doctype.owner.notnull()) - & (comment_doctype.owner != frappe.session.user) - & (comment_doctype.reference_owner == frappe.session.user) - & (comment_doctype.seen == 0) - ) - ) - - -def get_unread_emails(): - "returns count of unread emails for a user" - - communication_doctype = DocType("Communication") - user_doctype = DocType("User") - distinct_email_accounts = ( - frappe.qb.from_(user_doctype) - .select(user_doctype.email_account) - .where(user_doctype.parent == frappe.session.user) - .distinct() - ) - - return frappe.db.count(communication_doctype, - filters=( - (communication_doctype.communication_type == "Communication") - & (communication_doctype.communication_medium == "Email") - & (communication_doctype.sent_or_received == "Received") - & (communication_doctype.email_status.notin(["spam", "Trash"])) - & (communication_doctype.email_account.isin(distinct_email_accounts)) - & (communication_doctype.modified >= Now() - Interval(years=1)) - & (communication_doctype.seen == 0) - ) - ) diff --git a/frappe/core/page/dashboard_view/dashboard_view.js b/frappe/core/page/dashboard_view/dashboard_view.js index e8e9cc9502..bf9fb2a286 100644 --- a/frappe/core/page/dashboard_view/dashboard_view.js +++ b/frappe/core/page/dashboard_view/dashboard_view.js @@ -30,6 +30,7 @@ class Dashboard { show() { this.route = frappe.get_route(); + this.set_breadcrumbs(); if (this.route.length > 1) { // from route this.show_dashboard(this.route.slice(-1)[0]); @@ -75,6 +76,10 @@ class Dashboard { frappe.last_dashboard = current_dashboard_name; } + set_breadcrumbs() { + frappe.breadcrumbs.add("Desk", "Dashboard"); + } + refresh() { frappe.run_serially([ () => this.render_cards(), diff --git a/frappe/database/database.py b/frappe/database/database.py index 65242e0419..9fa1ff161c 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -10,19 +10,20 @@ import re import string from contextlib import contextmanager from time import time -from typing import Dict, List, Union, Tuple +from typing import Dict, List, Tuple, Union + +from pypika.terms import Criterion, NullValue, PseudoColumn import frappe import frappe.defaults import frappe.model.meta from frappe import _ -from frappe.utils import now, getdate, cast, get_datetime from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count -from frappe.query_builder.functions import Min, Max, Avg, Sum -from frappe.query_builder.utils import Column +from frappe.query_builder.utils import DocType +from frappe.utils import cast, get_datetime, getdate, now, sbool + from .query import Query -from pypika.terms import Criterion, PseudoColumn class Database(object): @@ -36,9 +37,9 @@ class Database(object): OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"] DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"] - STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype') - DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent', - 'parentfield', 'parenttype', 'idx'] + STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by') + DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'idx'] + CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield') MAX_WRITES_PER_TRANSACTION = 200_000 class InvalidColumnName(frappe.ValidationError): pass @@ -278,7 +279,9 @@ class Database(object): if self.auto_commit_on_many_writes: self.commit() else: - frappe.throw(_("Too many writes in one request. Please send smaller requests"), frappe.ValidationError) + msg = "

" + _("Too many changes to database in single action.") + "
" + msg += _("The changes have been reverted.") + "
" + raise frappe.TooManyWritesError(msg) def check_implicit_commit(self, query): if self.transaction_writes and \ @@ -432,11 +435,9 @@ class Database(object): else: fields = fieldname - if fieldname!="*": + if fieldname != "*": if isinstance(fieldname, str): fields = [fieldname] - else: - fields = fieldname if (filters is not None) and (filters!=doctype or doctype=="DocType"): try: @@ -555,7 +556,21 @@ class Database(object): def get_list(*args, **kwargs): return frappe.get_list(*args, **kwargs) - def get_single_value(self, doctype, fieldname, cache=False): + def set_single_value(self, doctype, fieldname, value, *args, **kwargs): + """Set field value of Single DocType. + + :param doctype: DocType of the single object + :param fieldname: `fieldname` of the property + :param value: `value` of the property + + Example: + + # Update the `deny_multiple_sessions` field in System Settings DocType. + company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True) + """ + return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs) + + def get_single_value(self, doctype, fieldname, cache=True): """Get property of Single DocType. Cache locally by default :param doctype: DocType of the single object whose value is requested @@ -570,7 +585,7 @@ class Database(object): if not doctype in self.value_cache: self.value_cache[doctype] = {} - if fieldname in self.value_cache[doctype]: + if cache and fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] val = self.query.get_sql( @@ -677,53 +692,55 @@ class Database(object): :param debug: Print the query in the developer / js console. :param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit. """ - if not modified: - modified = now() - if not modified_by: - modified_by = frappe.session.user + is_single_doctype = not (dn and dt != dn) + to_update = field if isinstance(field, dict) else {field: val} - to_update = {} if update_modified: - to_update = {"modified": modified, "modified_by": modified_by} + modified = modified or now() + modified_by = modified_by or frappe.session.user + to_update.update({"modified": modified, "modified_by": modified_by}) + + if is_single_doctype: + frappe.db.delete( + "Singles", + filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug + ) + + singles_data = ((dt, key, sbool(value)) for key, value in to_update.items()) + query = ( + frappe.qb.into("Singles") + .columns("doctype", "field", "value") + .insert(*singles_data) + ).run(debug=debug) + frappe.clear_document_cache(dt, dt) - if isinstance(field, dict): - to_update.update(field) else: - to_update.update({field: val}) + table = DocType(dt) - if dn and dt!=dn: - # with table - set_values = [] - for key in to_update: - set_values.append('`{0}`=%({0})s'.format(key)) + if for_update: + docnames = tuple( + self.get_values(dt, dn, "name", debug=debug, for_update=for_update, pluck=True) + ) or (NullValue(),) + query = frappe.qb.update(table).where(table.name.isin(docnames)) - for name in self.get_values(dt, dn, 'name', for_update=for_update, debug=debug): - values = dict(name=name[0]) - values.update(to_update) + for docname in docnames: + frappe.clear_document_cache(dt, docname) - self.sql("""update `tab{0}` - set {1} where name=%(name)s""".format(dt, ', '.join(set_values)), - values, debug=debug) + else: + query = self.query.build_conditions(table=dt, filters=dn, update=True) + # TODO: Fix this; doesn't work rn - gavin@frappe.io + # frappe.cache().hdel_keys(dt, "document_cache") + # Workaround: clear all document caches + frappe.cache().delete_value('document_cache') - frappe.clear_document_cache(dt, values['name']) - else: - # for singles - keys = list(to_update) - self.sql(''' - delete from `tabSingles` - where field in ({0}) and - doctype=%s'''.format(', '.join(['%s']*len(keys))), - list(keys) + [dt], debug=debug) - for key, value in to_update.items(): - self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''', - (dt, key, value), debug=debug) + for column, value in to_update.items(): + query = query.set(column, value) - frappe.clear_document_cache(dt, dn) + query.run(debug=debug) if dt in self.value_cache: del self.value_cache[dt] - @staticmethod def set(doc, field, val): """Set value in document. **Avoid**""" diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index cfb4e243a2..7c9309ee9f 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -171,9 +171,6 @@ CREATE TABLE `tabDocType` ( `modified_by` varchar(255) DEFAULT NULL, `owner` varchar(255) DEFAULT NULL, `docstatus` int(1) NOT NULL DEFAULT 0, - `parent` varchar(255) DEFAULT NULL, - `parentfield` varchar(255) DEFAULT NULL, - `parenttype` varchar(255) DEFAULT NULL, `idx` int(8) NOT NULL DEFAULT 0, `search_fields` varchar(255) DEFAULT NULL, `issingle` int(1) NOT NULL DEFAULT 0, @@ -228,8 +225,7 @@ CREATE TABLE `tabDocType` ( `subject_field` varchar(255) DEFAULT NULL, `sender_field` varchar(255) DEFAULT NULL, `migration_hash` varchar(255) DEFAULT NULL, - PRIMARY KEY (`name`), - KEY `parent` (`parent`) + PRIMARY KEY (`name`) ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 07bb4d5d7c..fd4bfc6dd0 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -18,6 +18,17 @@ class MariaDBTable(DBTable): if index_defs: additional_definitions += ',\n'.join(index_defs) + ',\n' + # child table columns + if self.meta.get("istable") or 0: + additional_definitions += ',\n'.join( + ( + f"parent varchar({varchar_len})", + f"parentfield varchar({varchar_len})", + f"parenttype varchar({varchar_len})", + "index parent(parent)" + ) + ) + ',\n' + # create table query = f"""create table `{self.table_name}` ( name varchar({varchar_len}) not null primary key, @@ -26,12 +37,8 @@ class MariaDBTable(DBTable): modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus int(1) not null default '0', - parent varchar({varchar_len}), - parentfield varchar({varchar_len}), - parenttype varchar({varchar_len}), idx int(8) not null default '0', {additional_definitions} - index parent(parent), index modified(modified)) ENGINE={engine} ROW_FORMAT=DYNAMIC diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index d5495c6879..a3266242a5 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -170,11 +170,11 @@ class PostgresDatabase(Database): @staticmethod def is_primary_key_violation(e): - return e.pgcode == '23505' and '_pkey' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == '23505' and '_pkey' in cstr(e.args[0]) @staticmethod def is_unique_key_violation(e): - return e.pgcode == '23505' and '_key' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == '23505' and '_key' in cstr(e.args[0]) @staticmethod def is_duplicate_fieldname(e): diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index f911e34650..1662b7b93e 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -176,9 +176,6 @@ CREATE TABLE "tabDocType" ( "modified_by" varchar(255) DEFAULT NULL, "owner" varchar(255) DEFAULT NULL, "docstatus" smallint NOT NULL DEFAULT 0, - "parent" varchar(255) DEFAULT NULL, - "parentfield" varchar(255) DEFAULT NULL, - "parenttype" varchar(255) DEFAULT NULL, "idx" bigint NOT NULL DEFAULT 0, "search_fields" varchar(255) DEFAULT NULL, "issingle" smallint NOT NULL DEFAULT 0, diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index a2d5be0b70..9487bc2fa7 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -5,26 +5,37 @@ from frappe.database.schema import DBTable, get_definition class PostgresTable(DBTable): def create(self): - add_text = '' + add_text = "" # columns column_defs = self.get_column_definitions() - if column_defs: add_text += ',\n'.join(column_defs) + if column_defs: + add_text += ",\n".join(column_defs) + + # child table columns + if self.meta.get("istable") or 0: + if column_defs: + add_text += ",\n" + + add_text += ",\n".join( + ( + "parent varchar({varchar_len})", + "parentfield varchar({varchar_len})", + "parenttype varchar({varchar_len})" + ) + ) # TODO: set docstatus length # create table - frappe.db.sql("""create table `%s` ( + frappe.db.sql(("""create table `%s` ( name varchar({varchar_len}) not null primary key, creation timestamp(6), modified timestamp(6), modified_by varchar({varchar_len}), owner varchar({varchar_len}), docstatus smallint not null default '0', - parent varchar({varchar_len}), - parentfield varchar({varchar_len}), - parenttype varchar({varchar_len}), idx bigint not null default '0', - %s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text)) + %s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN)) self.create_indexes() frappe.db.commit() diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 9a6dd502dc..dd54385c83 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -106,6 +106,9 @@ class DBTable: columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in frappe.db.STANDARD_VARCHAR_COLUMNS] + if self.meta.get("istable"): + columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in + frappe.db.CHILD_TABLE_COLUMNS] columns += self.columns.values() for col in columns: @@ -300,11 +303,12 @@ def validate_column_length(fieldname): def get_definition(fieldtype, precision=None, length=None): d = frappe.db.type_map.get(fieldtype) - # convert int to long int if the length of the int is greater than 11 - if fieldtype == "Int" and length and length > 11: - d = frappe.db.type_map.get("Long Int") + if not d: + return - if not d: return + if fieldtype == "Int" and length and length > 11: + # convert int to long int if the length of the int is greater than 11 + d = frappe.db.type_map.get("Long Int") coltype = d[0] size = d[1] if d[1] else None @@ -315,19 +319,44 @@ def get_definition(fieldtype, precision=None, length=None): if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6: size = '21,9' - if coltype == "varchar" and length: - size = length + if length: + if coltype == "varchar": + size = length + elif coltype == "int" and length < 11: + # allow setting custom length for int if length provided is less than 11 + # NOTE: this will only be applicable for mariadb as frappe implements int + # in postgres as bigint (as seen in type_map) + size = length if size is not None: coltype = "{coltype}({size})".format(coltype=coltype, size=size) return coltype -def add_column(doctype, column_name, fieldtype, precision=None): +def add_column( + doctype, + column_name, + fieldtype, + precision=None, + length=None, + default=None, + not_null=False +): if column_name in frappe.db.get_table_columns(doctype): # already exists return frappe.db.commit() - frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype, - column_name, get_definition(fieldtype, precision))) + + query = "alter table `tab%s` add column %s %s" % ( + doctype, + column_name, + get_definition(fieldtype, precision, length) + ) + + if not_null: + query += " not null" + if default: + query += f" default '{default}'" + + frappe.db.sql(query) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index a0523d90cd..20887f8886 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -42,13 +42,13 @@ def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None): doc = frappe.get_doc(doctype, d) try: message = '' - if action == 'submit' and doc.docstatus==0: + if action == 'submit' and doc.docstatus.is_draft(): doc.submit() message = _('Submiting {0}').format(doctype) - elif action == 'cancel' and doc.docstatus==1: + elif action == 'cancel' and doc.docstatus.is_submitted(): doc.cancel() message = _('Cancelling {0}').format(doctype) - elif action == 'update' and doc.docstatus < 2: + elif action == 'update' and not doc.docstatus.is_cancelled(): doc.update(data) doc.save() message = _('Updating {0}').format(doctype) diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index a49d5d5418..f0aa867c8a 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -52,3 +52,9 @@ def deferred_insert(routes): ] _deferred_insert("Route History", json.dumps(routes)) + +@frappe.whitelist() +def frequently_visited_links(): + return frappe.get_all('Route History', fields=['route', 'count(name) as count'], filters={ + 'user': frappe.session.user + }, group_by="route", order_by="count desc", limit=5) diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 381c24a765..d44c481210 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -148,8 +148,6 @@ def update_tags(doc, tags): "doctype": "Tag Link", "document_type": doc.doctype, "document_name": doc.name, - "parenttype": doc.doctype, - "parent": doc.name, "title": doc.get_title() or '', "tag": tag }).insert(ignore_permissions=True) diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index cd87c898d8..572d3f2a94 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -389,8 +389,6 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): else: return results - me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) - for dt, link in linkinfo.items(): filters = [] link["doctype"] = dt @@ -413,11 +411,16 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters")) elif link.get("get_parent"): - if me and me.parent and me.parenttype == dt: + ret = None + + # check for child table + if not frappe.get_meta(doctype).istable: + continue + + me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) + if me and me.parenttype == dt: ret = frappe.get_all(doctype=dt, fields=fields, filters=[[dt, "name", '=', me.parent]]) - else: - ret = None elif link.get("child_doctype"): or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")] @@ -473,7 +476,7 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled)) ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled)) - filters=[['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]] + filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters) @@ -498,12 +501,12 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): - filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]] + filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1) - links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1) + links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1) ret = {} @@ -529,34 +532,37 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False): ret = {} - filters=[['fieldtype','=', 'Dynamic Link']] + filters = [['fieldtype','=', 'Dynamic Link']] if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) # find dynamic links of parents links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) - links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) + links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) for df in links: if is_single(df.doctype): continue - # optimized to get both link exists and parenttype - possible_link = frappe.get_all(df.doctype, filters={df.doctype_fieldname: doctype}, - fields=['parenttype'], distinct=True) + is_child = frappe.get_meta(df.doctype).istable + possible_link = frappe.get_all( + df.doctype, + filters={df.doctype_fieldname: doctype}, + fields=["parenttype"] if is_child else None, + distinct=True + ) if not possible_link: continue - for d in possible_link: - # is child - if d.parenttype: + if is_child: + for d in possible_link: ret[d.parenttype] = { "child_doctype": df.doctype, "fieldname": [df.fieldname], "doctype_fieldname": df.doctype_fieldname } - else: - ret[df.doctype] = { - "fieldname": [df.fieldname], - "doctype_fieldname": df.doctype_fieldname - } + else: + ret[df.doctype] = { + "fieldname": [df.fieldname], + "doctype_fieldname": df.doctype_fieldname + } return ret diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 38b671d629..58d5b30103 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -91,8 +91,8 @@ def get_docinfo(doc=None, doctype=None, name=None): raise frappe.PermissionError all_communications = _get_communications(doc.doctype, doc.name) - automated_messages = filter(lambda x: x['communication_type'] == 'Automated Message', all_communications) - communications_except_auto_messages = filter(lambda x: x['communication_type'] != 'Automated Message', all_communications) + automated_messages = [msg for msg in all_communications if msg['communication_type'] == 'Automated Message'] + communications_except_auto_messages = [msg for msg in all_communications if msg['communication_type'] != 'Automated Message'] docinfo = frappe._dict(user_info = {}) @@ -119,6 +119,7 @@ def get_docinfo(doc=None, doctype=None, name=None): update_user_info(docinfo) frappe.response["docinfo"] = docinfo + return docinfo def add_comments(doc, docinfo): # divide comments into separate lists diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 27ac882016..c45fc9bfdd 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -6,7 +6,7 @@ import frappe, json import frappe.permissions from frappe.model.db_query import DatabaseQuery -from frappe.model import default_fields, optional_fields +from frappe.model import default_fields, optional_fields, child_table_fields from frappe import _ from io import StringIO from frappe.core.doctype.access_log.access_log import make_access_log @@ -156,7 +156,7 @@ def raise_invalid_field(fieldname): def is_standard(fieldname): if '.' in fieldname: parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None) - return fieldname in default_fields or fieldname in optional_fields + return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields def extract_fieldname(field): for text in (',', '/*', '#'): @@ -319,7 +319,7 @@ def export_query(): if add_totals_row: ret = append_totals_row(ret) - data = [['Sr'] + get_labels(db_query.fields, doctype)] + data = [[_('Sr')] + get_labels(db_query.fields, doctype)] for i, row in enumerate(ret): data.append([i+1] + list(row)) @@ -378,7 +378,8 @@ def get_labels(fields, doctype): for key in fields: key = key.split(" as ")[0] - if key.startswith(('count(', 'sum(', 'avg(')): continue + if key.startswith(('count(', 'sum(', 'avg(')): + continue if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") @@ -386,10 +387,16 @@ def get_labels(fields, doctype): parenttype = doctype fieldname = fieldname.strip("`") - df = frappe.get_meta(parenttype).get_field(fieldname) - label = df.label if df else fieldname.title() - if label in labels: - label = doctype + ": " + label + if parenttype == doctype and fieldname == "name": + label = _("ID", context="Label of name column in report") + else: + df = frappe.get_meta(parenttype).get_field(fieldname) + label = _(df.label if df else fieldname.title()) + if parenttype != doctype: + # If the column is from a child table, append the child doctype. + # For example, "Item Code (Sales Invoice Item)". + label += f" ({ _(parenttype) })" + labels.append(label) return labels diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index 34728375cd..682f0df7cf 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -252,7 +252,7 @@ def make_links(columns, data): if col.options and row.get(col.options): row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname]) elif col.fieldtype == "Currency": - doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.parent else None + doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.get("parent") else None # Pass the Document to get the currency based on docfield option row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc) return columns, data diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 3f8d399c52..3a1b683398 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -1,5 +1,6 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE + import email.utils import functools import imaplib @@ -7,6 +8,7 @@ import socket import time from datetime import datetime, timedelta from poplib import error_proto +from typing import List import frappe from frappe import _, are_emails_muted, safe_encode @@ -82,9 +84,6 @@ class EmailAccount(Document): if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - #if self.enable_incoming and not self.append_to: - # frappe.throw(_("Append To is mandatory for incoming mails")) - if (not self.awaiting_password and not frappe.local.flags.in_install and not frappe.local.flags.in_patch): if self.password or self.smtp_server in ('127.0.0.1', 'localhost'): @@ -422,10 +421,10 @@ class EmailAccount(Document): def get_failed_attempts_count(self): return cint(frappe.cache().get('{0}:email-account-failed-attempts'.format(self.name))) - def receive(self, test_mails=None): + def receive(self): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" exceptions = [] - inbound_mails = self.get_inbound_mails(test_mails=test_mails) + inbound_mails = self.get_inbound_mails() for mail in inbound_mails: try: communication = mail.process() @@ -442,7 +441,7 @@ class EmailAccount(Document): frappe.db.rollback() except Exception: frappe.db.rollback() - frappe.log_error('email_account.receive') + frappe.log_error(title="EmailAccount.receive") if self.use_imap: self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback()) exceptions.append(frappe.get_traceback()) @@ -458,20 +457,19 @@ class EmailAccount(Document): if exceptions: raise Exception(frappe.as_json(exceptions)) - def get_inbound_mails(self, test_mails=None): + def get_inbound_mails(self) -> List[InboundMail]: """retrive and return inbound mails. """ mails = [] - def process_mail(messages): + def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages['uid_list'][index] if messages.get('uid_list') else None - seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0 - mails.append(InboundMail(message, self, uid, seen_status)) - - if frappe.local.flags.in_test: - return [InboundMail(msg, self) for msg in test_mails or []] + seen_status = messages.get('seen_status', {}).get(uid) + if self.email_sync_option != 'UNSEEN' or seen_status != "SEEN": + # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' + mails.append(InboundMail(message, self, uid, seen_status, append_to)) if not self.enable_incoming: return [] @@ -482,10 +480,10 @@ class EmailAccount(Document): if self.use_imap: # process all given imap folder for folder in self.imap_folder: - email_server.select_imap_folder(folder.folder_name) - email_server.settings['uid_validity'] = folder.uidvalidity - messages = email_server.get_messages(folder=folder.folder_name) or {} - process_mail(messages) + if email_server.select_imap_folder(folder.folder_name): + email_server.settings['uid_validity'] = folder.uidvalidity + messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {} + process_mail(messages, folder.append_to) else: # process the pop3 account messages = email_server.get_messages() or {} @@ -495,7 +493,6 @@ class EmailAccount(Document): except Exception: frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) return [] - return mails def handle_bad_emails(self, uid, raw, reason): @@ -625,7 +622,6 @@ class EmailAccount(Document): if frappe.db.exists("Email Account", {"enable_automatic_linking": 1, "name": ('!=', self.name)}): frappe.throw(_("Automatic Linking can be activated only for one Email Account.")) - def append_email_to_sent_folder(self, message): email_server = None try: @@ -643,7 +639,8 @@ class EmailAccount(Document): message = safe_encode(message) email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message) except Exception: - frappe.log_error() + frappe.log_error(title="EmailAccount.append_email_to_sent_folder") + @frappe.whitelist() def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None): diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 6d26f9f070..f609c2947d 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -14,11 +14,11 @@ from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments from frappe.email.doctype.email_account.email_account import notify_unreplied +from unittest.mock import patch + make_test_records("User") make_test_records("Email Account") - - class TestEmailAccount(unittest.TestCase): @classmethod def setUpClass(cls): @@ -45,10 +45,21 @@ class TestEmailAccount(unittest.TestCase): def test_incoming(self): cleanup("test_sender@example.com") - test_mails = [self.get_test_mail('incoming-1.raw')] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-1.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) @@ -72,11 +83,21 @@ class TestEmailAccount(unittest.TestCase): existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile: - test_mails = [testfile.read()] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-2.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) @@ -93,11 +114,21 @@ class TestEmailAccount(unittest.TestCase): def test_incoming_attached_email_from_outlook_plain_text_only(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-3.raw"), "r") as f: - test_mails = [f.read()] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-3.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) @@ -106,11 +137,21 @@ class TestEmailAccount(unittest.TestCase): def test_incoming_attached_email_from_outlook_layers(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-4.raw"), "r") as f: - test_mails = [f.read()] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-4.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) @@ -151,11 +192,23 @@ class TestEmailAccount(unittest.TestCase): with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw"), "r") as f: raw = f.read() raw = raw.replace("<-- in-reply-to -->", sent_mail.get("Message-Id")) - test_mails = [raw] # parse reply + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + raw + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) sent = frappe.get_doc("Communication", sent_name) @@ -173,8 +226,20 @@ class TestEmailAccount(unittest.TestCase): test_mails.append(f.read()) # parse reply + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': test_mails, + 'seen_status': { + 2: 'UNSEEN', + 3: 'UNSEEN' + }, + 'uid_list': [2, 3] + } + } + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) @@ -197,11 +262,22 @@ class TestEmailAccount(unittest.TestCase): # get test mail with message-id as in-reply-to with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f: - test_mails = [f.read().replace('{{ message_id }}', last_mail.message_id)] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + f.read().replace('{{ message_id }}', last_mail.message_id) + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } # pull the mail email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) @@ -213,10 +289,21 @@ class TestEmailAccount(unittest.TestCase): def test_auto_reply(self): cleanup("test_sender@example.com") - test_mails = [self.get_test_mail('incoming-1.raw')] + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + self.get_test_mail('incoming-1.raw') + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.receive(test_mails=test_mails) + TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, @@ -246,6 +333,91 @@ class TestEmailAccount(unittest.TestCase): with self.assertRaises(Exception): email_account.validate() + def test_append_to(self): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + mail_content = self.get_test_mail(fname="incoming-2.raw") + + inbound_mail = InboundMail(mail_content, email_account, 12345, 1, 'ToDo') + communication = inbound_mail.process() + # the append_to for the email is set to ToDO in "_Test Email Account 1" + self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertTrue(communication.reference_name) + self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) + + def test_append_to_with_imap_folders(self): + mail_content_1 = self.get_test_mail(fname="incoming-1.raw") + mail_content_2 = self.get_test_mail(fname="incoming-2.raw") + mail_content_3 = self.get_test_mail(fname="incoming-3.raw") + + messages = { + # append_to = ToDo + '"INBOX"': { + 'latest_messages': [ + mail_content_1, + mail_content_2 + ], + 'seen_status': { + 0: 'UNSEEN', + 1: 'UNSEEN' + }, + 'uid_list': [0,1] + }, + # append_to = Communication + '"Test Folder"': { + 'latest_messages': [ + mail_content_3 + ], + 'seen_status': { + 2: 'UNSEEN' + }, + 'uid_list': [2] + } + } + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages) + self.assertEqual(len(mails), 3) + + inbox_mails = 0 + test_folder_mails = 0 + + for mail in mails: + communication = mail.process() + if mail.append_to == 'ToDo': + inbox_mails += 1 + self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertTrue(communication.reference_name) + self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) + else: + test_folder_mails += 1 + self.assertEqual(communication.reference_doctype, None) + + self.assertEqual(inbox_mails, 2) + self.assertEqual(test_folder_mails, 1) + + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) + @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) + def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + from frappe.email.receive import EmailServer + + def get_mocked_messages(**kwargs): + return messages.get(kwargs["folder"], {}) + + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): + mails = email_account.get_inbound_mails() + + return mails + + @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) + @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) + def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + def get_mocked_messages(**kwargs): + return messages.get(kwargs["folder"], {}) + + from frappe.email.receive import EmailServer + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): + email_account.receive() + class TestInboundMail(unittest.TestCase): @classmethod def setUpClass(cls): @@ -313,11 +485,11 @@ class TestInboundMail(unittest.TestCase): email_account = frappe.get_doc("Email Account", "_Test Email Account 1") inbound_mail = InboundMail(mail_content, email_account, 12345, 1) - new_communiction = inbound_mail.process() + new_communication = inbound_mail.process() # Make sure that uid is changed to new uid - self.assertEqual(new_communiction.uid, 12345) - self.assertEqual(communication.name, new_communiction.name) + self.assertEqual(new_communication.uid, 12345) + self.assertEqual(communication.name, new_communication.name) def test_find_parent_email_queue(self): """If the mail is reply to the already sent mail, there will be a email queue record. diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json index 450895d7a6..66eb5a9b2e 100644 --- a/frappe/email/doctype/email_account/test_records.json +++ b/frappe/email/doctype/email_account/test_records.json @@ -20,7 +20,7 @@ "pop3_server": "pop.test.example.com", "no_remaining":"0", "append_to": "ToDo", - "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], + "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}], "track_email_status": 1 }, { diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 9730004065..9b4f3b984c 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -11,7 +11,6 @@ import quopri from email.parser import Parser from email.policy import SMTPUTF8 from html2text import html2text -from six.moves import html_parser as HTMLParser import frappe from frappe import _, safe_encode, task @@ -20,6 +19,7 @@ from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message from frappe.email.email_body import add_attachment, get_formatted_html, get_email from frappe.utils import cint, split_emails, add_days, nowdate, cstr, get_hook_method from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.query_builder.utils import DocType MAX_RETRY_COUNT = 3 @@ -444,7 +444,7 @@ class QueueBuilder: try: text_content = html2text(self._message) - except HTMLParser.HTMLParseError: + except Exception: text_content = "See html attachment" return text_content + unsubscribe_text_message @@ -477,18 +477,27 @@ class QueueBuilder: all_ids = list(set(self.recipients + self.cc)) - EmailUnsubscribe = frappe.qb.DocType("Email Unsubscribe") + EmailUnsubscribe = DocType("Email Unsubscribe") + + if len(all_ids) > 0: + unsubscribed = ( + frappe.qb.from_(EmailUnsubscribe).select( + EmailUnsubscribe.email + ).where( + EmailUnsubscribe.email.isin(all_ids) + & ( + ( + (EmailUnsubscribe.reference_doctype == self.reference_doctype) + & (EmailUnsubscribe.reference_name == self.reference_name) + ) | ( + EmailUnsubscribe.global_unsubscribe == 1 + ) + ) + ).distinct() + ).run(pluck=True) + else: + unsubscribed = None - unsubscribed = (frappe.qb.from_(EmailUnsubscribe) - .select(EmailUnsubscribe.email) - .where(EmailUnsubscribe.email.isin(all_ids) & - ( - ( - (EmailUnsubscribe.reference_doctype == self.reference_doctype) & (EmailUnsubscribe.reference_name == self.reference_name) - ) | EmailUnsubscribe.global_unsubscribe == 1 - ) - ).distinct() - ).run(pluck=True) self._unsubscribed_user_emails = unsubscribed or [] return self._unsubscribed_user_emails diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 77979f9735..3fd96bdb6b 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -137,7 +137,7 @@ def get_context(context): if self.set_property_after_alert: allow_update = True - if doc.docstatus == 1 and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit: + if doc.docstatus.is_submitted() and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit: allow_update = False try: if allow_update and not doc.flags.in_notification_update: diff --git a/frappe/email/receive.py b/frappe/email/receive.py index dd64d0df80..b8156d5d9b 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -108,7 +108,8 @@ class EmailServer: raise def select_imap_folder(self, folder): - self.imap.select(folder) + res = self.imap.select(f'"{folder}"') + return res[0] == 'OK' # The folder exsits TODO: handle other resoponses too def logout(self): if cint(self.settings.use_imap): @@ -582,10 +583,11 @@ class Email: class InboundMail(Email): """Class representation of incoming mail along with mail handlers. """ - def __init__(self, content, email_account, uid=None, seen_status=None): + def __init__(self, content, email_account, uid=None, seen_status=None, append_to=None): super().__init__(content) self.email_account = email_account self.uid = uid or -1 + self.append_to = append_to self.seen_status = seen_status or 0 # System documents related to this mail @@ -623,15 +625,18 @@ class InboundMail(Email): if self.parent_communication(): data['in_reply_to'] = self.parent_communication().name + append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to + if self.reference_document(): data['reference_doctype'] = self.reference_document().doctype data['reference_name'] = self.reference_document().name - elif self.email_account.append_to and self.email_account.append_to != 'Communication': - reference_doc = self._create_reference_document(self.email_account.append_to) - if reference_doc: - data['reference_doctype'] = reference_doc.doctype - data['reference_name'] = reference_doc.name - data['is_first'] = True + else: + if append_to and append_to != 'Communication': + reference_doc = self._create_reference_document(append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True if self.is_notification(): # Disable notifications for notification. diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py index 8f1e5504da..0565b3219d 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py @@ -5,7 +5,7 @@ import frappe import json from frappe import _ from frappe.model.document import Document -from frappe.model import default_fields +from frappe.model import default_fields, child_table_fields class DocumentTypeMapping(Document): def validate(self): @@ -14,7 +14,7 @@ class DocumentTypeMapping(Document): def validate_inner_mapping(self): meta = frappe.get_meta(self.local_doctype) for field_map in self.field_mapping: - if field_map.local_fieldname not in default_fields: + if field_map.local_fieldname not in (default_fields + child_table_fields): field = meta.get_field(field_map.local_fieldname) if not field: frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx)) diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 8449425bc1..6ee72b5f81 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -103,6 +103,7 @@ class DocumentAlreadyRestored(ValidationError): pass class AttachmentLimitReached(ValidationError): pass class QueryTimeoutError(Exception): pass class QueryDeadlockError(Exception): pass +class TooManyWritesError(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 416d656d90..4242676d94 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -65,10 +65,7 @@ def get_latest_backup_file(with_files=False): return database, config -def get_file_size(file_path, unit): - if not unit: - unit = "MB" - +def get_file_size(file_path, unit='MB'): file_size = os.path.getsize(file_path) memory_size_unit_mapper = {"KB": 1, "MB": 2, "GB": 3, "TB": 4} @@ -99,7 +96,7 @@ def get_chunk_site(file_size): def validate_file_size(): frappe.flags.create_new_backup = True latest_file, site_config = get_latest_backup_file() - file_size = get_file_size(latest_file, unit="GB") + file_size = get_file_size(latest_file, unit="GB") if latest_file else 0 if file_size > 1: frappe.flags.create_new_backup = False diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index b50a0304a5..be9496c85b 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -90,11 +90,14 @@ default_fields = ( 'creation', 'modified', 'modified_by', + 'docstatus', + 'idx' +) + +child_table_fields = ( 'parent', 'parentfield', - 'parenttype', - 'idx', - 'docstatus' + 'parenttype' ) optional_fields = ( diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 11e97a38b9..307d95e84b 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -1,9 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE + import frappe import datetime from frappe import _ -from frappe.model import default_fields, table_fields +from frappe.model import default_fields, table_fields, child_table_fields from frappe.model.naming import set_new_name from frappe.model.utils.link_count import notify_link_count from frappe.modules import load_doctype_module @@ -11,6 +12,7 @@ from frappe.model import display_fieldtypes from frappe.utils import (cint, flt, now, cstr, strip_html, sanitize_html, sanitize_email, cast_fieldtype) from frappe.utils.html_utils import unescape_html +from frappe.model.docstatus import DocStatus max_positive_value = { 'smallint': 2 ** 15, @@ -20,6 +22,7 @@ max_positive_value = { DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link') + def get_controller(doctype): """Returns the **class** object of the given DocType. For `custom` type, returns `frappe.model.document.Document`. @@ -101,6 +104,10 @@ class BaseDocument(object): "balance": 42000 }) """ + + # QUESTION: why do we need the 1st for loop? + # we're essentially setting the values in d, in the 2nd for loop (?) + # first set default field values of base document for key in default_fields: if key in d: @@ -205,7 +212,10 @@ class BaseDocument(object): raise ValueError def remove(self, doc): - self.get(doc.parentfield).remove(doc) + # Usage: from the parent doc, pass the child table doc + # to remove that child doc from the child table, thus removing it from the parent doc + if doc.get("parentfield"): + self.get(doc.parentfield).remove(doc) def _init_child(self, value, key): if not self.doctype: @@ -224,7 +234,7 @@ class BaseDocument(object): value.parentfield = key if value.docstatus is None: - value.docstatus = 0 + value.docstatus = DocStatus.draft() if not getattr(value, "idx", None): value.idx = len(self.get(key) or []) + 1 @@ -282,8 +292,11 @@ class BaseDocument(object): if key not in self.__dict__: self.__dict__[key] = None - if key in ("idx", "docstatus") and self.__dict__[key] is None: - self.__dict__[key] = 0 + if self.__dict__[key] is None: + if key == "docstatus": + self.docstatus = DocStatus.draft() + elif key == "idx": + self.__dict__[key] = 0 for key in self.get_valid_columns(): if key not in self.__dict__: @@ -304,12 +317,27 @@ class BaseDocument(object): def is_new(self): return self.get("__islocal") - def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False): + @property + def docstatus(self): + return DocStatus(self.get("docstatus")) + + @docstatus.setter + def docstatus(self, value): + self.__dict__["docstatus"] = DocStatus(cint(value)) + + def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False): doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str) doc["doctype"] = self.doctype for df in self.meta.get_table_fields(): children = self.get(df.fieldname) or [] - doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children] + doc[df.fieldname] = [ + d.as_dict( + convert_dates_to_str=convert_dates_to_str, + no_nulls=no_nulls, + no_default_fields=no_default_fields, + no_child_table_fields=no_child_table_fields + ) for d in children + ] if no_nulls: for k in list(doc): @@ -321,6 +349,11 @@ class BaseDocument(object): if k in default_fields: del doc[k] + if no_child_table_fields: + for k in list(doc): + if k in child_table_fields: + del doc[k] + for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"): if self.get(key): doc[key] = self.get(key) @@ -492,7 +525,7 @@ class BaseDocument(object): self.set(df.fieldname, flt(self.get(df.fieldname))) if self.docstatus is not None: - self.docstatus = cint(self.docstatus) + self.docstatus = DocStatus(cint(self.docstatus)) def _get_missing_mandatory_fields(self): """Get mandatory fields that do not have any values""" @@ -500,12 +533,12 @@ class BaseDocument(object): if df.fieldtype in table_fields: return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label)) - elif self.parentfield: + # check if parentfield exists (only applicable for child table doctype) + elif self.get("parentfield"): return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)), _("Row"), self.idx, _("Value missing for"), _(df.label)) - else: - return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label)) + return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label)) missing = [] @@ -524,10 +557,11 @@ class BaseDocument(object): def get_invalid_links(self, is_submittable=False): """Returns list of invalid links and also updates fetch values if not set""" def get_msg(df, docname): - if self.parentfield: + # check if parentfield exists (only applicable for child table doctype) + if self.get("parentfield"): return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname) - else: - return "{}: {}".format(_(df.label), docname) + + return "{}: {}".format(_(df.label), docname) invalid_links = [] cancelled_links = [] @@ -581,7 +615,7 @@ class BaseDocument(object): setattr(self, df.fieldname, values.name) for _df in fields_to_fetch: - if self.is_new() or self.docstatus != 1 or _df.allow_on_submit: + if self.is_new() or not self.docstatus.is_submitted() or _df.allow_on_submit: self.set_fetch_from_value(doctype, _df, values) notify_link_count(doctype, docname) @@ -591,7 +625,7 @@ class BaseDocument(object): elif (df.fieldname != "amended_from" and (is_submittable or self.meta.is_submittable) and frappe.get_meta(doctype).is_submittable - and cint(frappe.db.get_value(doctype, docname, "docstatus"))==2): + and cint(frappe.db.get_value(doctype, docname, "docstatus")) == DocStatus.cancelled()): cancelled_links.append((df.fieldname, docname, get_msg(df, docname))) @@ -601,11 +635,8 @@ class BaseDocument(object): fetch_from_fieldname = df.fetch_from.split('.')[-1] value = values[fetch_from_fieldname] if df.fieldtype in ['Small Text', 'Text', 'Data']: - if fetch_from_fieldname in default_fields: - from frappe.model.meta import get_default_df - fetch_from_df = get_default_df(fetch_from_fieldname) - else: - fetch_from_df = frappe.get_meta(doctype).get_field(fetch_from_fieldname) + from frappe.model.meta import get_default_df + fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname) if not fetch_from_df: frappe.throw( @@ -740,9 +771,9 @@ class BaseDocument(object): def throw_length_exceeded_error(self, df, max_length, value): - if self.parentfield and self.idx: + # check if parentfield exists (only applicable for child table doctype) + if self.get("parentfield"): reference = _("{0}, Row {1}").format(_(self.doctype), self.idx) - else: reference = "{0} {1}".format(_(self.doctype), self.name) @@ -805,8 +836,8 @@ class BaseDocument(object): or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode", "Code") # cancelled and submit but not update after submit should be ignored - or self.docstatus==2 - or (self.docstatus==1 and not df.get("allow_on_submit"))): + or self.docstatus.is_cancelled() + or (self.docstatus.is_submitted() and not df.get("allow_on_submit"))): continue else: @@ -853,7 +884,7 @@ class BaseDocument(object): :param parentfield: If fieldname is in child table.""" from frappe.model.meta import get_field_precision - if parentfield and not isinstance(parentfield, str): + if parentfield and not isinstance(parentfield, str) and parentfield.get("parentfield"): parentfield = parentfield.parentfield cache_key = parentfield or "main" @@ -880,7 +911,7 @@ class BaseDocument(object): from frappe.utils.formatters import format_value df = self.meta.get_field(fieldname) - if not df and fieldname in default_fields: + if not df: from frappe.model.meta import get_default_df df = get_default_df(fieldname) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 2fddcf9e33..2cc99575d6 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -212,7 +212,7 @@ def check_permission_and_not_submitted(doc): .format(doc.doctype, doc.name), raise_exception=frappe.PermissionError) # check if submitted - if doc.docstatus == 1: + if doc.docstatus.is_submitted(): frappe.msgprint(_("{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.").format(_(doc.doctype), doc.name, "", ""), raise_exception=True) @@ -222,32 +222,35 @@ def check_if_doc_is_linked(doc, method="Delete"): """ from frappe.model.rename_doc import get_link_fields link_fields = get_link_fields(doc.doctype) - link_fields = [[lf['parent'], lf['fieldname'], lf['issingle']] for lf in link_fields] + ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + + for lf in link_fields: + link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle'] - for link_dt, link_field, issingle in link_fields: if not issingle: - for item in frappe.db.get_values(link_dt, {link_field:doc.name}, - ["name", "parent", "parenttype", "docstatus"], as_dict=True): - linked_doctype = item.parenttype if item.parent else link_dt + fields = ["name", "docstatus"] + if frappe.get_meta(link_dt).istable: + fields.extend(["parent", "parenttype"]) - ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True): + # available only in child table cases + item_parent = getattr(item, "parent", None) + linked_doctype = item.parenttype if item_parent else link_dt if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'): # don't check for communication and todo! continue - if not item: - continue - elif method != "Delete" and (method != "Cancel" or item.docstatus != 1): + if method != "Delete" and (method != "Cancel" or item.docstatus != 1): # don't raise exception if not # linked to a non-cancelled doc when deleting or to a submitted doc when cancelling continue - elif link_dt == doc.doctype and (item.parent or item.name) == doc.name: + elif link_dt == doc.doctype and (item_parent or item.name) == doc.name: # don't raise exception if not # linked to same item or doc having same name as the item continue else: - reference_docname = item.parent or item.name + reference_docname = item_parent or item.name raise_link_exists_exception(doc, linked_doctype, reference_docname) else: diff --git a/frappe/model/docstatus.py b/frappe/model/docstatus.py new file mode 100644 index 0000000000..01aab1e491 --- /dev/null +++ b/frappe/model/docstatus.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. See LICENSE + + +class DocStatus(int): + def is_draft(self): + return self == self.draft() + + def is_submitted(self): + return self == self.submitted() + + def is_cancelled(self): + return self == self.cancelled() + + @classmethod + def draft(cls): + return cls(0) + + @classmethod + def submitted(cls): + return cls(1) + + @classmethod + def cancelled(cls): + return cls(2) diff --git a/frappe/model/document.py b/frappe/model/document.py index e482be0056..66a0cef7dd 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,13 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe +import hashlib +import json import time +from werkzeug.exceptions import NotFound + +import frappe from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc, validate_name -from werkzeug.exceptions import NotFound, Forbidden -import hashlib, json +from frappe.model.naming import set_new_name, validate_name +from frappe.model.docstatus import DocStatus from frappe.model import optional_fields, table_fields from frappe.model.workflow import validate_workflow from frappe.model.workflow import set_workflow_state_on_action @@ -17,6 +20,7 @@ from frappe.desk.form.document_follow import follow_document from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event from frappe.utils.data import get_absolute_url + # once_only validation # methods @@ -307,9 +311,6 @@ class Document(BaseDocument): self.check_permission("write", "save") - if self.docstatus == 2: - self._rename_doc_on_cancel() - self.set_user_and_timestamp() self.set_docstatus() self.check_if_latest() @@ -474,7 +475,7 @@ class Document(BaseDocument): # We'd probably want the creation and owner to be set via API # or Data import at some point, that'd have to be handled here - if self.is_new(): + if self.is_new() and not (frappe.flags.in_patch or frappe.flags.in_migrate): self.creation = self.modified self.owner = self.modified_by @@ -490,7 +491,7 @@ class Document(BaseDocument): def set_docstatus(self): if self.docstatus is None: - self.docstatus=0 + self.docstatus = DocStatus.draft() for d in self.get_all_children(): d.docstatus = self.docstatus @@ -526,7 +527,7 @@ class Document(BaseDocument): def _validate_non_negative(self): def get_msg(df): - if self.parentfield: + if self.get("parentfield"): return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)), _("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label))) else: @@ -720,6 +721,7 @@ class Document(BaseDocument): else: tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` where name = %s for update""".format(self.doctype), self.name, as_dict=True) + if not tmp: frappe.throw(_("Record does not exist")) else: @@ -740,7 +742,7 @@ class Document(BaseDocument): else: self.check_docstatus_transition(0) - def check_docstatus_transition(self, docstatus): + def check_docstatus_transition(self, to_docstatus): """Ensures valid `docstatus` transition. Valid transitions are (number in brackets is `docstatus`): @@ -751,31 +753,32 @@ class Document(BaseDocument): """ if not self.docstatus: - self.docstatus = 0 - if docstatus==0: - if self.docstatus==0: + self.docstatus = DocStatus.draft() + + if to_docstatus == DocStatus.draft(): + if self.docstatus.is_draft(): self._action = "save" - elif self.docstatus==1: + elif self.docstatus.is_submitted(): self._action = "submit" self.check_permission("submit") - elif self.docstatus==2: + elif self.docstatus.is_cancelled(): raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)")) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) - elif docstatus==1: - if self.docstatus==1: + elif to_docstatus == DocStatus.submitted(): + if self.docstatus.is_submitted(): self._action = "update_after_submit" self.check_permission("submit") - elif self.docstatus==2: + elif self.docstatus.is_cancelled(): self._action = "cancel" self.check_permission("cancel") - elif self.docstatus==0: + elif self.docstatus.is_draft(): raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 (Submitted) to 0 (Draft)")) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) - elif docstatus==2: + elif to_docstatus == DocStatus.cancelled(): raise frappe.ValidationError(_("Cannot edit cancelled document")) def set_parent_in_children(self): @@ -929,14 +932,14 @@ class Document(BaseDocument): @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" - self.docstatus = 1 + self.docstatus = DocStatus.submitted() return self.save() @whitelist.__func__ def _cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves. """ - self.docstatus = 2 + self.docstatus = DocStatus.cancelled() return self.save() @whitelist.__func__ @@ -954,7 +957,7 @@ class Document(BaseDocument): frappe.delete_doc(self.doctype, self.name, ignore_permissions = ignore_permissions, flags=self.flags) def run_before_save_methods(self): - """Run standard methods before `INSERT` or `UPDATE`. Standard Methods are: + """Run standard methods before `INSERT` or `UPDATE`. Standard Methods are: - `validate`, `before_save` for **Save**. - `validate`, `before_submit` for **Submit**. @@ -1199,7 +1202,7 @@ class Document(BaseDocument): if not frappe.compare(val1, condition, val2): label = doc.meta.get_label(fieldname) condition_str = error_condition_map.get(condition, condition) - if doc.parentfield: + if doc.get("parentfield"): msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2) else: msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2) @@ -1223,7 +1226,7 @@ class Document(BaseDocument): doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]})) for fieldname in fieldnames: - doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield))) + doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield")))) def get_url(self): """Returns Desk URL for this document.""" @@ -1371,19 +1374,16 @@ class Document(BaseDocument): from frappe.desk.doctype.tag.tag import DocTags return DocTags(self.doctype).get_tags(self.name).split(",")[1:] - def _rename_doc_on_cancel(self): - new_name = gen_new_name_for_cancelled_doc(self) - frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) - self.name = new_name - def __repr__(self): name = self.name or "unsaved" doctype = self.__class__.__name__ docstatus = f" docstatus={self.docstatus}" if self.docstatus else "" - parent = f" parent={self.parent}" if self.parent else "" + repr_str = f"<{doctype}: {name}{docstatus}" - return f"<{doctype}: {name}{docstatus}{parent}>" + if not hasattr(self, "parent"): + return repr_str + ">" + return f"{repr_str} parent={self.parent}>" def __str__(self): name = self.name or "unsaved" diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index bde4fb6d73..f40a43bb73 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -4,7 +4,7 @@ import json import frappe from frappe import _ -from frappe.model import default_fields, table_fields +from frappe.model import default_fields, table_fields, child_table_fields from frappe.utils import cstr @@ -149,6 +149,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent): no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] + [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] + list(default_fields) + + list(child_table_fields) + list(table_map.get("field_no_map", []))) for df in target_doc.meta.get("fields"): diff --git a/frappe/model/meta.py b/frappe/model/meta.py index a483f3f2d6..372392f689 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -18,7 +18,7 @@ from datetime import datetime import click import frappe, json, os from frappe.utils import cstr, cint, cast -from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields +from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields, child_table_fields from frappe.model.document import Document from frappe.model.base_document import BaseDocument from frappe.modules import load_doctype_module @@ -191,6 +191,8 @@ class Meta(Document): else: self._valid_columns = self.default_fields + \ [df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes] + if self.istable: + self._valid_columns += list(child_table_fields) return self._valid_columns @@ -520,7 +522,7 @@ class Meta(Document): '''add `links` child table in standard link dashboard format''' dashboard_links = [] - if hasattr(self, 'links') and self.links: + if getattr(self, 'links', None): dashboard_links.extend(self.links) if not data.transactions: @@ -625,9 +627,9 @@ def get_field_currency(df, doc=None): frappe.local.field_currency = frappe._dict() if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or - (doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))): + (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))): - ref_docname = doc.parent or doc.name + ref_docname = doc.get("parent") or doc.name if ":" in cstr(df.get("options")): split_opts = df.get("options").split(":") @@ -635,7 +637,7 @@ def get_field_currency(df, doc=None): currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2]) else: currency = doc.get(df.get("options")) - if doc.parent: + if doc.get("parenttype"): if currency: ref_docname = doc.name else: @@ -648,7 +650,7 @@ def get_field_currency(df, doc=None): .setdefault(df.fieldname, currency) return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \ - (doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname)) + (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname)) def get_field_precision(df, doc=None, currency=None): """get precision based on DocField options and fieldvalue in doc""" @@ -669,19 +671,25 @@ def get_field_precision(df, doc=None, currency=None): def get_default_df(fieldname): - if fieldname in default_fields: + if fieldname in (default_fields + child_table_fields): if fieldname in ("creation", "modified"): return frappe._dict( fieldname = fieldname, fieldtype = "Datetime" ) - else: + elif fieldname in ("idx", "docstatus"): return frappe._dict( fieldname = fieldname, - fieldtype = "Data" + fieldtype = "Int" ) + return frappe._dict( + fieldname = fieldname, + fieldtype = "Data" + ) + + def trim_tables(doctype=None, dry_run=False, quiet=False): """ Removes database fields that don't exist in the doctype (json or custom field). This may be needed @@ -713,7 +721,7 @@ def trim_tables(doctype=None, dry_run=False, quiet=False): def trim_table(doctype, dry_run=True): frappe.cache().hdel('table_columns', f"tab{doctype}") - ignore_fields = default_fields + optional_fields + ignore_fields = default_fields + optional_fields + child_table_fields columns = frappe.db.get_table_columns(doctype) fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value() is_internal = lambda f: f not in ignore_fields and not f.startswith("_") diff --git a/frappe/model/naming.py b/frappe/model/naming.py index f3d68f3715..b2d11a4cfc 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,14 +1,3 @@ -"""utilities to generate a document name based on various rules defined. - -NOTE: -Till version 13, whenever a submittable document is amended it's name is set to orig_name-X, -where X is a counter and it increments when amended again and so on. - -From Version 14, The naming pattern is changed in a way that amended documents will -have the original name `orig_name` instead of `orig_name-X`. To make this happen -the cancelled document naming pattern is changed to 'orig_name-CANC-X'. -""" - # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE @@ -40,7 +29,7 @@ def set_new_name(doc): doc.name = None if getattr(doc, "amended_from", None): - doc.name = _get_amended_name(doc) + _set_amended_name(doc) return elif getattr(doc.meta, "issingle", False): @@ -256,18 +245,6 @@ def revert_series_if_last(key, name, doc=None): * prefix = #### and hashes = 2021 (hash doesn't exist) * will search hash in key then accordingly get prefix = "" """ - if hasattr(doc, 'amended_from'): - # Do not revert the series if the document is amended. - if doc.amended_from: - return - - # Get document name by parsing incase of fist cancelled document - if doc.docstatus == 2 and not doc.amended_from: - if doc.name.endswith('-CANC'): - name, _ = NameParser.parse_docname(doc.name, sep='-CANC') - else: - name, _ = NameParser.parse_docname(doc.name, sep='-CANC-') - if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: @@ -356,9 +333,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" return value -def _get_amended_name(doc): - name, _ = NameParser(doc).parse_amended_from() - return name +def _set_amended_name(doc): + am_id = 1 + am_prefix = doc.amended_from + if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): + am_id = cint(doc.amended_from.split("-")[-1]) + 1 + am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen + + doc.name = am_prefix + "-" + str(am_id) + return doc.name + def _field_autoname(autoname, doc, skip_slicing=None): """ @@ -399,83 +383,3 @@ def _format_autoname(autoname, doc): name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) return name - -class NameParser: - """Parse document name and return parts of it. - - NOTE: It handles cancellend and amended doc parsing for now. It can be expanded. - """ - def __init__(self, doc): - self.doc = doc - - def parse_amended_from(self): - """ - Cancelled document naming will be in one of these formats - - * original_name-X-CANC - This is introduced to migrate old style naming to new style - * original_name-CANC - This is introduced to migrate old style naming to new style - * original_name-CANC-X - This is the new style naming - - New style naming: In new style naming amended documents will have original name. That says, - when a document gets cancelled we need rename the document by adding `-CANC-X` to the end - so that amended documents can use the original name. - - Old style naming: cancelled documents stay with original name and when amended, amended one - gets a new name as `original_name-X`. To bring new style naming we had to change the existing - cancelled document names and that is done by adding `-CANC` to cancelled documents through patch. - """ - if not getattr(self.doc, 'amended_from', None): - return (None, None) - - # Handle old style cancelled documents (original_name-X-CANC, original_name-CANC) - if self.doc.amended_from.endswith('-CANC'): - name, _ = self.parse_docname(self.doc.amended_from, '-CANC') - amended_from_doc = frappe.get_all( - self.doc.doctype, - filters = {'name': self.doc.amended_from}, - fields = ['amended_from'], - limit=1) - - # Handle format original_name-X-CANC. - if amended_from_doc and amended_from_doc[0].amended_from: - return self.parse_docname(name, '-') - return name, None - - # Handle new style cancelled documents - return self.parse_docname(self.doc.amended_from, '-CANC-') - - @classmethod - def parse_docname(cls, name, sep='-'): - split_list = name.rsplit(sep, 1) - - if len(split_list) == 1: - return (name, None) - return (split_list[0], split_list[1]) - -def get_cancelled_doc_latest_counter(tname, docname): - """Get the latest counter used for cancelled docs of given docname. - """ - name_prefix = f'{docname}-CANC-' - - rows = frappe.db.sql(""" - select - name - from `tab{tname}` - where - name like %(name_prefix)s and docstatus=2 - """.format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1) - - if not rows: - return -1 - return max([int(row.name.replace(name_prefix, '') or -1) for row in rows]) - -def gen_new_name_for_cancelled_doc(doc): - """Generate a new name for cancelled document. - """ - if getattr(doc, "amended_from", None): - name, _ = NameParser(doc).parse_amended_from() - else: - name = doc.name - - counter = get_cancelled_doc_latest_counter(doc.doctype, name) - return f'{name}-CANC-{counter+1}' diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index e74d88c0f2..1b26cc2c3a 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -1,10 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json import frappe -from frappe.utils import cint from frappe import _ -import json +from frappe.utils import cint +from frappe.model.docstatus import DocStatus class WorkflowStateError(frappe.ValidationError): pass class WorkflowTransitionError(frappe.ValidationError): pass @@ -102,13 +103,13 @@ def apply_workflow(doc, action): doc.set(next_state.update_field, next_state.update_value) new_docstatus = cint(next_state.doc_status) - if doc.docstatus == 0 and new_docstatus == 0: + if doc.docstatus.is_draft() and new_docstatus == DocStatus.draft(): doc.save() - elif doc.docstatus == 0 and new_docstatus == 1: + elif doc.docstatus.is_draft() and new_docstatus == DocStatus.submitted(): doc.submit() - elif doc.docstatus == 1 and new_docstatus == 1: + elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.submitted(): doc.save() - elif doc.docstatus == 1 and new_docstatus == 2: + elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.cancelled(): doc.cancel() else: frappe.throw(_('Illegal Document Status for {0}').format(next_state.state)) @@ -212,10 +213,10 @@ def bulk_workflow_approval(docnames, doctype, action): frappe.db.commit() except Exception as e: if not frappe.message_log: - # Exception is raised manually and not from msgprint or throw + # Exception is raised manually and not from msgprint or throw message = "{0}".format(e.__class__.__name__) if e.args: - message += " : {0}".format(e.args[0]) + message += " : {0}".format(e.args[0]) message_dict = {"docname": docname, "message": message} failed_transactions[docname].append(message_dict) diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py index ab6ffd4985..45e008fa04 100644 --- a/frappe/modules/export_file.py +++ b/frappe/modules/export_file.py @@ -47,7 +47,7 @@ def strip_default_fields(doc, doc_export): for df in doc.meta.get_table_fields(): for d in doc_export.get(df.fieldname): - for fieldname in frappe.model.default_fields: + for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): if fieldname in d: del d[fieldname] diff --git a/frappe/patches.txt b/frappe/patches.txt index c393b456e3..db9610a767 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -119,7 +119,6 @@ execute:frappe.delete_doc_if_exists('DocType', 'GSuite Settings') execute:frappe.delete_doc_if_exists('DocType', 'GSuite Templates') execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Account') execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings') -frappe.patches.v12_0.remove_parent_and_parenttype_from_print_formats frappe.patches.v12_0.remove_example_email_thread_notify execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders() frappe.patches.v12_0.set_correct_url_in_files @@ -175,6 +174,7 @@ execute:frappe.delete_doc_if_exists('Page', 'workspace') execute:frappe.delete_doc_if_exists('Page', 'dashboard', force=1) frappe.core.doctype.page.patches.drop_unused_pages execute:frappe.get_doc('Role', 'Guest').save() # remove desk access +frappe.patches.v13_0.remove_chat frappe.patches.v13_0.rename_desk_page_to_workspace # 02.02.2021 frappe.patches.v13_0.delete_package_publish_tool frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings @@ -184,10 +184,10 @@ frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty frappe.patches.v13_0.set_first_day_of_the_week -frappe.patches.v14_0.rename_cancelled_documents frappe.patches.v14_0.update_workspace2 # 20.09.2021 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021 frappe.patches.v14_0.transform_todo_schema +frappe.patches.v14_0.remove_post_and_post_comment [post_model_sync] frappe.patches.v14_0.drop_data_import_legacy diff --git a/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py b/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py deleted file mode 100644 index 1a3c56da59..0000000000 --- a/frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py +++ /dev/null @@ -1,14 +0,0 @@ -import frappe - -def execute(): - frappe.db.sql(""" - UPDATE - `tabPrint Format` - SET - `tabPrint Format`.`parent`='', - `tabPrint Format`.`parenttype`='', - `tabPrint Format`.parentfield='' - WHERE - `tabPrint Format`.parent != '' - OR `tabPrint Format`.parenttype != '' - """) \ No newline at end of file diff --git a/frappe/patches/v13_0/remove_chat.py b/frappe/patches/v13_0/remove_chat.py new file mode 100644 index 0000000000..1804c7693f --- /dev/null +++ b/frappe/patches/v13_0/remove_chat.py @@ -0,0 +1,17 @@ +import frappe +import click + +def execute(): + frappe.delete_doc_if_exists("DocType", "Chat Message") + frappe.delete_doc_if_exists("DocType", "Chat Message Attachment") + frappe.delete_doc_if_exists("DocType", "Chat Profile") + frappe.delete_doc_if_exists("DocType", "Chat Token") + frappe.delete_doc_if_exists("DocType", "Chat Room User") + frappe.delete_doc_if_exists("DocType", "Chat Room") + frappe.delete_doc_if_exists("Module Def", "Chat") + + click.secho( + "Chat Module is moved to a separate app and is removed from Frappe in version-13.\n" + "Please install the app to continue using the chat feature: https://github.com/frappe/chat", + fg="yellow", + ) \ No newline at end of file diff --git a/frappe/patches/v14_0/remove_post_and_post_comment.py b/frappe/patches/v14_0/remove_post_and_post_comment.py new file mode 100644 index 0000000000..3a93139961 --- /dev/null +++ b/frappe/patches/v14_0/remove_post_and_post_comment.py @@ -0,0 +1,5 @@ +import frappe + +def execute(): + frappe.delete_doc_if_exists("DocType", "Post") + frappe.delete_doc_if_exists("DocType", "Post Comment") diff --git a/frappe/patches/v14_0/rename_cancelled_documents.py b/frappe/patches/v14_0/rename_cancelled_documents.py deleted file mode 100644 index 4b565d4f76..0000000000 --- a/frappe/patches/v14_0/rename_cancelled_documents.py +++ /dev/null @@ -1,213 +0,0 @@ -import functools -import traceback - -import frappe - -def execute(): - """Rename cancelled documents by adding a postfix. - """ - rename_cancelled_docs() - -def get_submittable_doctypes(): - """Returns list of submittable doctypes in the system. - """ - return frappe.db.get_all('DocType', filters={'is_submittable': 1}, pluck='name') - -def get_cancelled_doc_names(doctype): - """Return names of cancelled document names those are in old format. - """ - docs = frappe.db.get_all(doctype, filters={'docstatus': 2}, pluck='name') - return [each for each in docs if not (each.endswith('-CANC') or ('-CANC-' in each))] - -@functools.lru_cache() -def get_linked_doctypes(): - """Returns list of doctypes those are linked with given doctype using 'Link' fieldtype. - """ - filters=[['fieldtype','=', 'Link']] - links = frappe.get_all("DocField", - fields=["parent", "fieldname", "options as linked_to"], - filters=filters, - as_list=1) - - links+= frappe.get_all("Custom Field", - fields=["dt as parent", "fieldname", "options as linked_to"], - filters=filters, - as_list=1) - - links_by_doctype = {} - for doctype, fieldname, linked_to in links: - links_by_doctype.setdefault(linked_to, []).append((doctype, fieldname)) - return links_by_doctype - -@functools.lru_cache() -def get_single_doctypes(): - return frappe.get_all("DocType", filters={'issingle': 1}, pluck='name') - -@functools.lru_cache() -def get_dynamic_linked_doctypes(): - filters=[['fieldtype','=', 'Dynamic Link']] - - # find dynamic links of parents - links = frappe.get_all("DocField", - fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], - filters=filters, - as_list=1) - links+= frappe.get_all("Custom Field", - fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], - filters=filters, - as_list=1) - return links - -@functools.lru_cache() -def get_child_tables(): - """ - """ - filters =[['fieldtype', 'in', ('Table', 'Table MultiSelect')]] - links = frappe.get_all("DocField", - fields=["parent as doctype", "options as child_table"], - filters=filters, - as_list=1) - - links+= frappe.get_all("Custom Field", - fields=["dt as doctype", "options as child_table"], - filters=filters, - as_list=1) - - map = {} - for doctype, child_table in links: - map.setdefault(doctype, []).append(child_table) - return map - -def update_cancelled_document_names(doctype, cancelled_doc_names): - return frappe.db.sql(""" - update - `tab{doctype}` - set - name=CONCAT(name, '-CANC') - where - docstatus=2 - and - name in %(cancelled_doc_names)s; - """.format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names}) - -def update_amended_field(doctype, cancelled_doc_names): - return frappe.db.sql(""" - update - `tab{doctype}` - set - amended_from=CONCAT(amended_from, '-CANC') - where - amended_from in %(cancelled_doc_names)s; - """.format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names}) - -def update_attachments(doctype, cancelled_doc_names): - frappe.db.sql(""" - update - `tabFile` - set - attached_to_name=CONCAT(attached_to_name, '-CANC') - where - attached_to_doctype=%(dt)s and attached_to_name in %(cancelled_doc_names)s - """, {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - -def update_versions(doctype, cancelled_doc_names): - frappe.db.sql(""" - UPDATE - `tabVersion` - SET - docname=CONCAT(docname, '-CANC') - WHERE - ref_doctype=%(dt)s AND docname in %(cancelled_doc_names)s - """, {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - -def update_linked_doctypes(doctype, cancelled_doc_names): - single_doctypes = get_single_doctypes() - - for linked_dt, field in get_linked_doctypes().get(doctype, []): - if linked_dt not in single_doctypes: - frappe.db.sql(""" - update - `tab{linked_dt}` - set - `{column}`=CONCAT(`{column}`, '-CANC') - where - `{column}` in %(cancelled_doc_names)s; - """.format(linked_dt=linked_dt, column=field), - {'cancelled_doc_names': cancelled_doc_names}) - else: - doc = frappe.get_single(linked_dt) - if getattr(doc, field) in cancelled_doc_names: - setattr(doc, field, getattr(doc, field)+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def update_dynamic_linked_doctypes(doctype, cancelled_doc_names): - single_doctypes = get_single_doctypes() - - for linked_dt, fieldname, doctype_fieldname in get_dynamic_linked_doctypes(): - if linked_dt not in single_doctypes: - frappe.db.sql(""" - update - `tab{linked_dt}` - set - `{column}`=CONCAT(`{column}`, '-CANC') - where - `{column}` in %(cancelled_doc_names)s and {doctype_fieldname}=%(dt)s; - """.format(linked_dt=linked_dt, column=fieldname, doctype_fieldname=doctype_fieldname), - {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - else: - doc = frappe.get_single(linked_dt) - if getattr(doc, doctype_fieldname) == doctype and getattr(doc, fieldname) in cancelled_doc_names: - setattr(doc, fieldname, getattr(doc, fieldname)+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def update_child_tables(doctype, cancelled_doc_names): - child_tables = get_child_tables().get(doctype, []) - single_doctypes = get_single_doctypes() - - for table in child_tables: - if table not in single_doctypes: - frappe.db.sql(""" - update - `tab{table}` - set - parent=CONCAT(parent, '-CANC') - where - parenttype=%(dt)s and parent in %(cancelled_doc_names)s; - """.format(table=table), {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - else: - doc = frappe.get_single(table) - if getattr(doc, 'parenttype')==doctype and getattr(doc, 'parent') in cancelled_doc_names: - setattr(doc, 'parent', getattr(doc, 'parent')+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def rename_cancelled_docs(): - submittable_doctypes = get_submittable_doctypes() - - for dt in submittable_doctypes: - for retry in range(2): - try: - cancelled_doc_names = tuple(get_cancelled_doc_names(dt)) - if not cancelled_doc_names: - break - update_cancelled_document_names(dt, cancelled_doc_names) - update_amended_field(dt, cancelled_doc_names) - update_child_tables(dt, cancelled_doc_names) - update_linked_doctypes(dt, cancelled_doc_names) - update_dynamic_linked_doctypes(dt, cancelled_doc_names) - update_attachments(dt, cancelled_doc_names) - update_versions(dt, cancelled_doc_names) - print(f"Renaming cancelled records of {dt} doctype") - frappe.db.commit() - break - except Exception: - if retry == 1: - print(f"Failed to rename the cancelled records of {dt} doctype, moving on!") - traceback.print_exc() - frappe.db.rollback() - diff --git a/frappe/public/js/frappe-web.bundle.js b/frappe/public/js/frappe-web.bundle.js index c962457964..b8d4006090 100644 --- a/frappe/public/js/frappe-web.bundle.js +++ b/frappe/public/js/frappe-web.bundle.js @@ -2,6 +2,7 @@ import "./jquery-bootstrap"; import "./frappe/class.js"; import "./frappe/polyfill.js"; import "./lib/md5.min.js"; +import "./lib/moment.js"; import "./frappe/provide.js"; import "./frappe/format.js"; import "./frappe/utils/number_format.js"; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 2264042539..b153718c70 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -331,7 +331,7 @@ frappe.data_import.ImportPreview = class ImportPreview { is_row_imported(row) { let serial_no = row[0].content; return this.import_log.find(log => { - return log.success && log.row_indexes.includes(serial_no); + return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no); }); } }; diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index 167b4955fa..1b30726a7a 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -534,22 +534,21 @@ export default { }); }, show_google_drive_picker() { - let dialog = cur_dialog; - dialog.hide(); + this.close_dialog = true; let google_drive = new GoogleDrivePicker({ - pickerCallback: data => this.google_drive_callback(data, dialog), + pickerCallback: data => this.google_drive_callback(data), ...this.google_drive_settings }); google_drive.loadPicker(); }, - google_drive_callback(data, dialog) { + google_drive_callback(data) { if (data.action == google.picker.Action.PICKED) { this.upload_file({ file_url: data.docs[0].url, file_name: data.docs[0].name }); } else if (data.action == google.picker.Action.CANCEL) { - dialog.show(); + cur_frm.attachments.new_attachment() } }, url_to_file(url, filename, mime_type) { diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index ed355cf8b4..9f02485a9e 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -374,10 +374,22 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat } set_custom_query(args) { - var set_nulls = function(obj) { - $.each(obj, function(key, value) { - if(value!==undefined) { - obj[key] = value; + const is_valid_value = (value, key) => { + if (value) return true; + // check if empty value is valid + if (this.frm) { + let field = frappe.meta.get_docfield(this.frm.doctype, key); + // empty value link fields is invalid + return !field || !["Link", "Dynamic Link"].includes(field.fieldtype); + } else { + return value !== undefined; + } + } + + const set_nulls = (obj) => { + $.each(obj, (key, value) => { + if (!is_valid_value(value, key)) { + delete obj[key]; } }); return obj; @@ -458,7 +470,6 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat validate_link_and_fetch(df, options, docname, value) { if (!options) return; - let field_value = ""; const fetch_map = this.fetch_map; const columns_to_fetch = Object.values(fetch_map); @@ -467,16 +478,10 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat return value; } - return frappe.xcall("frappe.client.validate_link", { - doctype: options, - docname: value, - fields: columns_to_fetch, - }).then((response) => { - if (!docname || !columns_to_fetch.length) return response.name; - + function update_dependant_fields(response) { + let field_value = ""; for (const [target_field, source_field] of Object.entries(fetch_map)) { if (value) field_value = response[source_field]; - frappe.model.set_value( df.parent, docname, @@ -485,9 +490,23 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat df.fieldtype, ); } + } - return response.name; - }); + // to avoid unnecessary request + if (value) { + return frappe.xcall("frappe.client.validate_link", { + doctype: options, + docname: value, + fields: columns_to_fetch, + }).then((response) => { + if (!docname || !columns_to_fetch.length) return response.name; + update_dependant_fields(response); + return response.name; + }); + } else { + update_dependant_fields({}); + return value; + } } get fetch_map() { diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 5c0b6b1399..c0c7ce8b4e 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -860,36 +860,32 @@ frappe.ui.form.Form = class FrappeForm { } _cancel(btn, callback, on_error, skip_confirm) { + const me = this; const cancel_doc = () => { frappe.validated = true; - this.script_manager.trigger("before_cancel").then(() => { + me.script_manager.trigger("before_cancel").then(() => { if (!frappe.validated) { - return this.handle_save_fail(btn, on_error); + return me.handle_save_fail(btn, on_error); } - const original_name = this.docname; - const after_cancel = (r) => { + var after_cancel = function(r) { if (r.exc) { - this.handle_save_fail(btn, on_error); + me.handle_save_fail(btn, on_error); } else { frappe.utils.play_sound("cancel"); + me.refresh(); callback && callback(); - this.script_manager.trigger("after_cancel"); - frappe.run_serially([ - () => this.rename_notify(this.doctype, original_name, r.docs[0].name), - () => frappe.router.clear_re_route(this.doctype, original_name), - () => this.refresh(), - ]); + me.script_manager.trigger("after_cancel"); } }; - frappe.ui.form.save(this, "cancel", after_cancel, btn); + frappe.ui.form.save(me, "cancel", after_cancel, btn); }); } if (skip_confirm) { cancel_doc(); } else { - frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error)); + frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error)); } }; @@ -911,7 +907,7 @@ frappe.ui.form.Form = class FrappeForm { 'docname': this.doc.name }).then(is_amended => { if (is_amended) { - frappe.throw(__('This document is already amended, you cannot amend it again')); + frappe.throw(__('This document is already amended, you cannot ammend it again')); } this.validate_form_action("Amend"); var me = this; diff --git a/frappe/public/js/frappe/form/multi_select_dialog.js b/frappe/public/js/frappe/form/multi_select_dialog.js index bc0286e62d..bc5f7a9b52 100644 --- a/frappe/public/js/frappe/form/multi_select_dialog.js +++ b/frappe/public/js/frappe/form/multi_select_dialog.js @@ -150,8 +150,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { }); } + is_child_selection_enabled() { + return this.dialog.fields_dict['allow_child_item_selection'].get_value(); + } + toggle_child_selection() { - if (this.dialog.fields_dict['allow_child_item_selection'].get_value()) { + if (this.is_child_selection_enabled()) { this.show_child_results(); } else { this.child_results = []; @@ -289,7 +293,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { parent: this.dialog.get_field('filter_area').$wrapper, doctype: this.doctype, on_change: () => { - this.get_results(); + if (this.is_child_selection_enabled()) { + this.show_child_results(); + } else { + this.get_results(); + } } }); // 'Apply Filter' breaks since the filers are not in a popover @@ -325,7 +333,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { this.$parent.find('.input-with-feedback').on('change', () => { frappe.flags.auto_scroll = false; - this.get_results(); + if (this.is_child_selection_enabled()) { + this.show_child_results(); + } else { + this.get_results(); + } }); this.$parent.find('[data-fieldtype="Data"]').on('input', () => { @@ -333,8 +345,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { clearTimeout($this.data('timeout')); $this.data('timeout', setTimeout(function () { frappe.flags.auto_scroll = false; - me.empty_list(); - me.get_results(); + if (me.is_child_selection_enabled()) { + me.show_child_results(); + } else { + me.empty_list(); + me.get_results(); + } }, 300)); }); } diff --git a/frappe/public/js/frappe/form/script_manager.js b/frappe/public/js/frappe/form/script_manager.js index d1732ee702..6169fa75b8 100644 --- a/frappe/public/js/frappe/form/script_manager.js +++ b/frappe/public/js/frappe/form/script_manager.js @@ -192,7 +192,7 @@ frappe.ui.form.ScriptManager = class ScriptManager { } function setup_add_fetch(df) { - if ((['Data', 'Read Only', 'Text', 'Small Text', 'Currency', 'Check', + if ((['Data', 'Read Only', 'Text', 'Small Text', 'Currency', 'Check', 'Attach Image', 'Text Editor', 'Code', 'Link', 'Float', 'Int', 'Date', 'Select', 'Duration'].includes(df.fieldtype) || df.read_only==1) && df.fetch_from && df.fetch_from.indexOf(".")!=-1) { var parts = df.fetch_from.split("."); diff --git a/frappe/public/js/frappe/form/templates/form_sidebar.html b/frappe/public/js/frappe/form/templates/form_sidebar.html index efa8b9ea5e..dcea2f4647 100644 --- a/frappe/public/js/frappe/form/templates/form_sidebar.html +++ b/frappe/public/js/frappe/form/templates/form_sidebar.html @@ -1,5 +1,5 @@ -