${m.message}
` : '';
+ return title + message;
+ })
+ .join('');
+ let id = frappe.dom.get_unique_id();
+ html = `${messages}
+
+
+
`;
+ }
+ let indicator_color = log.success ? 'green' : 'red';
+ let title = log.success ? __('Success') : __('Failure');
+
+ if (frm.doc.show_failed_logs && log.success) {
+ return '';
+ }
+
+ return `
+
+ ${log.exception}
+ ${title}
+ | ${__('Row Number')} | +${__('Status')} | +${__('Message')} | +
|---|
${m.message}
` : '';
- return title + message;
- })
- .join('');
- let id = frappe.dom.get_unique_id();
- html = `${messages}
-
-
-
`;
+ frm.toggle_display('import_log_section', false);
+ frm.add_custom_button(__('Export Import Log'), () =>
+ frm.trigger('export_import_log')
+ );
}
- let indicator_color = log.success ? 'green' : 'red';
- let title = log.success ? __('Success') : __('Failure');
-
- if (frm.doc.show_failed_logs && log.success) {
- return '';
- }
-
- return `
-
- ${log.exception}
- ${title}
- | ${__('Row Number')} | -${__('Status')} | -${__('Message')} | -
|---|
Or
" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file\n", - "description": "Must be a publicly accessible Google Sheets URL", - "fieldname": "google_sheets_url", - "fieldtype": "Data", - "label": "Import from Google Sheets", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", - "fieldname": "refresh_google_sheet", - "fieldtype": "Button", - "label": "Refresh Google Sheet" - } - ], - "hide_toolbar": 1, - "links": [], - "modified": "2021-04-11 01:50:42.074623", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 + "actions": [], + "autoname": "format:{reference_doctype} Import on {creation}", + "beta": 1, + "creation": "2019-08-04 14:16:08.318714", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "import_type", + "download_template", + "import_file", + "payload_count", + "html_5", + "google_sheets_url", + "refresh_google_sheet", + "column_break_5", + "status", + "submit_after_import", + "mute_emails", + "template_options", + "import_warnings_section", + "template_warnings", + "import_warnings", + "section_import_preview", + "import_preview", + "import_log_section", + "show_failed_logs", + "import_log_preview" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "import_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Import Type", + "options": "\nInsert New Records\nUpdate Existing Records", + "reqd": 1, + "set_only_once": 1 + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "import_file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Import File", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "fieldname": "import_preview", + "fieldtype": "HTML", + "label": "Import Preview" + }, + { + "fieldname": "section_import_preview", + "fieldtype": "Section Break", + "label": "Preview" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "template_options", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Options", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "import_log_section", + "fieldtype": "Section Break", + "label": "Import Log" + }, + { + "fieldname": "import_log_preview", + "fieldtype": "HTML", + "label": "Import Log Preview" + }, + { + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "label": "Status", + "options": "Pending\nSuccess\nPartial Success\nError", + "read_only": 1 + }, + { + "fieldname": "template_warnings", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Warnings", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "label": "Submit After Import", + "set_only_once": 1 + }, + { + "fieldname": "import_warnings_section", + "fieldtype": "Section Break", + "label": "Import File Errors and Warnings" + }, + { + "fieldname": "import_warnings", + "fieldtype": "HTML", + "label": "Import Warnings" + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" + }, + { + "default": "1", + "fieldname": "mute_emails", + "fieldtype": "Check", + "label": "Don't Send Emails", + "set_only_once": 1 + }, + { + "default": "0", + "fieldname": "show_failed_logs", + "fieldtype": "Check", + "label": "Show Failed Logs" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file", + "fieldname": "html_5", + "fieldtype": "HTML", + "options": "Or
" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file\n", + "description": "Must be a publicly accessible Google Sheets URL", + "fieldname": "google_sheets_url", + "fieldtype": "Data", + "label": "Import from Google Sheets", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", + "fieldname": "refresh_google_sheet", + "fieldtype": "Button", + "label": "Refresh Google Sheet" + }, + { + "fieldname": "payload_count", + "fieldtype": "Int", + "hidden": 1, + "label": "Payload Count", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "links": [], + "modified": "2022-02-01 20:08:37.624914", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 5935ddc4ba..5972e79b4d 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -27,6 +27,7 @@ class DataImport(Document): self.validate_import_file() self.validate_google_sheets_url() + self.set_payload_count() def validate_import_file(self): if self.import_file: @@ -38,6 +39,12 @@ class DataImport(Document): return validate_google_sheets_url(self.google_sheets_url) + def set_payload_count(self): + if self.import_file: + i = self.get_importer() + payloads = i.import_file.get_payloads_for_import() + self.payload_count = len(payloads) + @frappe.whitelist() def get_preview_from_template(self, import_file=None, google_sheets_url=None): if import_file: @@ -67,7 +74,7 @@ class DataImport(Document): enqueue( start_import, queue="default", - timeout=6000, + timeout=10000, event="data_import", job_name=self.name, data_import=self.name, @@ -80,6 +87,9 @@ class DataImport(Document): def export_errored_rows(self): return self.get_importer().export_errored_rows() + def download_import_log(self): + return self.get_importer().export_import_log() + def get_importer(self): return Importer(self.reference_doctype, data_import=self) @@ -90,7 +100,6 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N import_file, google_sheets_url ) - @frappe.whitelist() def form_start_import(data_import): return frappe.get_doc("Data Import", data_import).start_import() @@ -145,6 +154,30 @@ def download_errored_template(data_import_name): data_import = frappe.get_doc("Data Import", data_import_name) data_import.export_errored_rows() +@frappe.whitelist() +def download_import_log(data_import_name): + data_import = frappe.get_doc("Data Import", data_import_name) + data_import.download_import_log() + +@frappe.whitelist() +def get_import_status(data_import_name): + import_status = {} + + logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'], + filters={'data_import': data_import_name}, + group_by='success') + + total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count') + + for log in logs: + if log.get('success'): + import_status['success'] = log.get('count') + else: + import_status['failed'] = log.get('count') + + import_status['total_records'] = total_payload_count + + return import_status def import_file( doctype, file_path, import_type, submit_after_import=False, console=False diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 0eb05aa354..6ab750ba25 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -24,12 +24,14 @@ frappe.listview_settings['Data Import'] = { 'Error': 'red' }; let status = doc.status; + if (imports_in_progress.includes(doc.name)) { status = 'In Progress'; } if (status == 'Pending') { status = 'Not Started'; } + return [__(status), colors[status], 'status,=,' + doc.status]; }, formatters: { diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index b9b2050763..107c05a66a 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -47,7 +47,13 @@ class Importer: ) def get_data_for_import_preview(self): - return self.import_file.get_data_for_import_preview() + out = self.import_file.get_data_for_import_preview() + + out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index", limit=10) + + return out def before_import(self): # set user lang for translations @@ -58,7 +64,6 @@ class Importer: frappe.flags.in_import = True frappe.flags.mute_emails = self.data_import.mute_emails - self.data_import.db_set("status", "Pending") self.data_import.db_set("template_warnings", "") def import_data(self): @@ -79,20 +84,25 @@ class Importer: return # setup import log - if self.data_import.import_log: - import_log = frappe.parse_json(self.data_import.import_log) - else: - import_log = [] + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] - # remove previous failures from import log - import_log = [log for log in import_log if log.get("success")] + log_index = 0 + + # Do not remove rows in case of retry after an error or pending data import + if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count: + # remove previous failures from import log only in case of retry after partial success + import_log = [log for log in import_log if log.get("success")] # get successfully imported rows imported_rows = [] for log in import_log: log = frappe._dict(log) - if log.success: - imported_rows += log.row_indexes + if log.success or len(import_log) < self.data_import.payload_count: + imported_rows += json.loads(log.row_indexes) + + log_index = log.log_index # start import total_payload_count = len(payloads) @@ -146,25 +156,41 @@ class Importer: }, ) - import_log.append( - frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) - ) + create_import_log(self.data_import.name, log_index, { + 'success': True, + 'docname': doc.name, + 'row_indexes': row_indexes + }) + + log_index += 1 + + if not self.data_import.status == "Partial Success": + self.data_import.db_set("status", "Partial Success") + # commit after every successful import frappe.db.commit() except Exception: - import_log.append( - frappe._dict( - success=False, - exception=frappe.get_traceback(), - messages=frappe.local.message_log, - row_indexes=row_indexes, - ) - ) + messages = frappe.local.message_log frappe.clear_messages() + # rollback if exception frappe.db.rollback() + create_import_log(self.data_import.name, log_index, { + 'success': False, + 'exception': frappe.get_traceback(), + 'messages': messages, + 'row_indexes': row_indexes + }) + + log_index += 1 + + # Logs are db inserted directly so will have to be fetched again + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + # set status failures = [log for log in import_log if not log.get("success")] if len(failures) == total_payload_count: @@ -178,7 +204,6 @@ class Importer: self.print_import_log(import_log) else: self.data_import.db_set("status", status) - self.data_import.db_set("import_log", json.dumps(import_log)) self.after_import() @@ -248,11 +273,14 @@ class Importer: if not self.data_import: return - import_log = frappe.parse_json(self.data_import.import_log or "[]") + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + failures = [log for log in import_log if not log.get("success")] row_indexes = [] for f in failures: - row_indexes.extend(f.get("row_indexes", [])) + row_indexes.extend(json.loads(f.get("row_indexes", []))) # de duplicate row_indexes = list(set(row_indexes)) @@ -264,6 +292,30 @@ class Importer: build_csv_response(rows, _(self.doctype)) + def export_import_log(self): + from frappe.utils.csvutils import build_csv_response + + if not self.data_import: + return + + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": self.data_import.name}, + order_by="log_index") + + header_row = ["Row Numbers", "Status", "Message", "Exception"] + + rows = [header_row] + + for log in import_log: + row_number = json.loads(log.get("row_indexes"))[0] + status = "Success" if log.get('success') else "Failure" + message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \ + log.get("messages") + exception = frappe.utils.cstr(log.get("exception", '')) + rows += [[row_number, status, message, exception]] + + build_csv_response(rows, self.doctype) + def print_import_log(self, import_log): failed_records = [log for log in import_log if not log.success] successful_records = [log for log in import_log if log.success] @@ -1172,3 +1224,17 @@ def df_as_json(df): def get_select_options(df): return [d for d in (df.options or "").split("\n") if d] + +def create_import_log(data_import, log_index, log_details): + frappe.get_doc({ + 'doctype': 'Data Import Log', + 'log_index': log_index, + 'success': log_details.get('success'), + 'data_import': data_import, + 'row_indexes': json.dumps(log_details.get('row_indexes')), + 'docname': log_details.get('docname'), + 'messages': json.dumps(log_details.get('messages', '[]')), + 'exception': log_details.get('exception') + }).db_insert() + + diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index e1bc0e7ca5..3f78594dd2 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -60,15 +60,19 @@ class TestImporter(unittest.TestCase): frappe.local.message_log = [] data_import.start_import() data_import.reload() - import_log = frappe.parse_json(data_import.import_log) - self.assertEqual(import_log[0]['row_indexes'], [2,3]) - expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error) - expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error) - self.assertEqual(import_log[1]['row_indexes'], [4]) - self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required") + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": data_import.name}, + order_by="log_index") + + self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3]) + expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error) + expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error) + + self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4]) + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required") def test_data_import_update(self): existing_doc = frappe.get_doc( diff --git a/frappe/social/doctype/post/__init__.py b/frappe/core/doctype/data_import_log/__init__.py similarity index 100% rename from frappe/social/doctype/post/__init__.py rename to frappe/core/doctype/data_import_log/__init__.py diff --git a/frappe/core/doctype/data_import_log/data_import_log.js b/frappe/core/doctype/data_import_log/data_import_log.js new file mode 100644 index 0000000000..c376edeec9 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2021, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Data Import Log', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/data_import_log/data_import_log.json b/frappe/core/doctype/data_import_log/data_import_log.json new file mode 100644 index 0000000000..b1d991f099 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.json @@ -0,0 +1,84 @@ +{ + "actions": [], + "creation": "2021-12-25 16:12:20.205889", + "doctype": "DocType", + "editable_grid": 1, + "engine": "MyISAM", + "field_order": [ + "data_import", + "row_indexes", + "success", + "docname", + "messages", + "exception", + "log_index" + ], + "fields": [ + { + "fieldname": "data_import", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Data Import", + "options": "Data Import" + }, + { + "fieldname": "docname", + "fieldtype": "Data", + "label": "Reference Name" + }, + { + "fieldname": "exception", + "fieldtype": "Text", + "label": "Exception" + }, + { + "fieldname": "row_indexes", + "fieldtype": "Code", + "label": "Row Indexes", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "success", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Success" + }, + { + "fieldname": "log_index", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Log Index" + }, + { + "fieldname": "messages", + "fieldtype": "Code", + "label": "Messages", + "options": "JSON" + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2021-12-29 11:19:19.646076", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import Log", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC" +} \ No newline at end of file diff --git a/frappe/core/doctype/data_import_log/data_import_log.py b/frappe/core/doctype/data_import_log/data_import_log.py new file mode 100644 index 0000000000..a71aefa8bc --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class DataImportLog(Document): + pass diff --git a/frappe/core/doctype/data_import_log/test_data_import_log.py b/frappe/core/doctype/data_import_log/test_data_import_log.py new file mode 100644 index 0000000000..244404936e --- /dev/null +++ b/frappe/core/doctype/data_import_log/test_data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and Contributors +# See license.txt + +# import frappe +import unittest + +class TestDataImportLog(unittest.TestCase): + pass diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3754288145..67c31b704d 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -699,6 +699,13 @@ class DocType(Document): if not name: name = self.name + # a Doctype name is the tablename created in database + # `tab" + _("Too many changes to database in single action.") + "
" + msg += _("The changes have been reverted.") + "
" + raise frappe.TooManyWritesError(msg) def check_implicit_commit(self, query): if self.transaction_writes and \ @@ -555,7 +558,21 @@ class Database(object): def get_list(*args, **kwargs): return frappe.get_list(*args, **kwargs) - def get_single_value(self, doctype, fieldname, cache=False): + def set_single_value(self, doctype, fieldname, value, *args, **kwargs): + """Set field value of Single DocType. + + :param doctype: DocType of the single object + :param fieldname: `fieldname` of the property + :param value: `value` of the property + + Example: + + # Update the `deny_multiple_sessions` field in System Settings DocType. + company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True) + """ + return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs) + + def get_single_value(self, doctype, fieldname, cache=True): """Get property of Single DocType. Cache locally by default :param doctype: DocType of the single object whose value is requested @@ -570,7 +587,7 @@ class Database(object): if not doctype in self.value_cache: self.value_cache[doctype] = {} - if fieldname in self.value_cache[doctype]: + if cache and fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] val = self.query.get_sql( @@ -677,53 +694,55 @@ class Database(object): :param debug: Print the query in the developer / js console. :param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit. """ - if not modified: - modified = now() - if not modified_by: - modified_by = frappe.session.user + is_single_doctype = not (dn and dt != dn) + to_update = field if isinstance(field, dict) else {field: val} - to_update = {} if update_modified: - to_update = {"modified": modified, "modified_by": modified_by} + modified = modified or now() + modified_by = modified_by or frappe.session.user + to_update.update({"modified": modified, "modified_by": modified_by}) + + if is_single_doctype: + frappe.db.delete( + "Singles", + filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug + ) + + singles_data = ((dt, key, sbool(value)) for key, value in to_update.items()) + query = ( + frappe.qb.into("Singles") + .columns("doctype", "field", "value") + .insert(*singles_data) + ).run(debug=debug) + frappe.clear_document_cache(dt, dt) - if isinstance(field, dict): - to_update.update(field) else: - to_update.update({field: val}) + table = DocType(dt) - if dn and dt!=dn: - # with table - set_values = [] - for key in to_update: - set_values.append('`{0}`=%({0})s'.format(key)) + if for_update: + docnames = tuple( + self.get_values(dt, dn, "name", debug=debug, for_update=for_update, pluck=True) + ) or (NullValue(),) + query = frappe.qb.update(table).where(table.name.isin(docnames)) - for name in self.get_values(dt, dn, 'name', for_update=for_update, debug=debug): - values = dict(name=name[0]) - values.update(to_update) + for docname in docnames: + frappe.clear_document_cache(dt, docname) - self.sql("""update `tab{0}` - set {1} where name=%(name)s""".format(dt, ', '.join(set_values)), - values, debug=debug) + else: + query = self.query.build_conditions(table=dt, filters=dn, update=True) + # TODO: Fix this; doesn't work rn - gavin@frappe.io + # frappe.cache().hdel_keys(dt, "document_cache") + # Workaround: clear all document caches + frappe.cache().delete_value('document_cache') - frappe.clear_document_cache(dt, values['name']) - else: - # for singles - keys = list(to_update) - self.sql(''' - delete from `tabSingles` - where field in ({0}) and - doctype=%s'''.format(', '.join(['%s']*len(keys))), - list(keys) + [dt], debug=debug) - for key, value in to_update.items(): - self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''', - (dt, key, value), debug=debug) + for column, value in to_update.items(): + query = query.set(column, value) - frappe.clear_document_cache(dt, dn) + query.run(debug=debug) if dt in self.value_cache: del self.value_cache[dt] - @staticmethod def set(doc, field, val): """Set value in document. **Avoid**""" diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 6b827a4e89..de28dad900 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -245,9 +245,16 @@ class MariaDBDatabase(Database): column_name as 'name', column_type as 'type', column_default as 'default', - column_key = 'MUL' as 'index', + COALESCE( + (select 1 + from information_schema.statistics + where table_name="{table_name}" + and column_name=columns.column_name + and NON_UNIQUE=1 + limit 1 + ), 0) as 'index', column_key = 'UNI' as 'unique' - from information_schema.columns + from information_schema.columns as columns where table_name = '{table_name}' '''.format(table_name=table_name), as_dict=1) def has_index(self, table_name, index_name): diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 5768a2f23d..07bb4d5d7c 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -58,18 +58,34 @@ class MariaDBTable(DBTable): modify_column_query.append("MODIFY `{}` {}".format(col.fieldname, col.get_definition())) for col in self.add_index: - # if index key not exists - if not frappe.db.sql("SHOW INDEX FROM `%s` WHERE key_name = %s" % - (self.table_name, '%s'), col.fieldname): - add_index_query.append("ADD INDEX `{}`(`{}`)".format(col.fieldname, col.fieldname)) + # if index key does not exists + if not frappe.db.has_index(self.table_name, col.fieldname + '_index'): + add_index_query.append("ADD INDEX `{}_index`(`{}`)".format(col.fieldname, col.fieldname)) - for col in self.drop_index: + for col in self.drop_index + self.drop_unique: if col.fieldname != 'name': # primary key + current_column = self.current_columns.get(col.fieldname.lower()) + unique_constraint_changed = current_column.unique != col.unique + if unique_constraint_changed and not col.unique: + # nosemgrep + unique_index_record = frappe.db.sql(""" + SHOW INDEX FROM `{0}` + WHERE Key_name=%s + AND Non_unique=0 + """.format(self.table_name), (col.fieldname), as_dict=1) + if unique_index_record: + drop_index_query.append("DROP INDEX `{}`".format(unique_index_record[0].Key_name)) + index_constraint_changed = current_column.index != col.set_index # if index key exists - if frappe.db.sql("""SHOW INDEX FROM `{0}` - WHERE key_name=%s - AND Non_unique=%s""".format(self.table_name), (col.fieldname, col.unique)): - drop_index_query.append("drop index `{}`".format(col.fieldname)) + if index_constraint_changed and not col.set_index: + # nosemgrep + index_record = frappe.db.sql(""" + SHOW INDEX FROM `{0}` + WHERE Key_name=%s + AND Non_unique=1 + """.format(self.table_name), (col.fieldname + '_index'), as_dict=1) + if index_record: + drop_index_query.append("DROP INDEX `{}`".format(index_record[0].Key_name)) try: for query_parts in [add_column_query, modify_column_query, add_index_query, drop_index_query]: diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 33f07990af..a3266242a5 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -77,11 +77,11 @@ class PostgresDatabase(Database): """Escape quotes and percent in given string.""" if isinstance(s, bytes): s = s.decode('utf-8') - + # MariaDB's driver treats None as an empty string # So Postgres should do the same - if s is None: + if s is None: s = '' if percent: @@ -170,11 +170,11 @@ class PostgresDatabase(Database): @staticmethod def is_primary_key_violation(e): - return e.pgcode == '23505' and '_pkey' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == '23505' and '_pkey' in cstr(e.args[0]) @staticmethod def is_unique_key_violation(e): - return e.pgcode == '23505' and '_key' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == '23505' and '_key' in cstr(e.args[0]) @staticmethod def is_duplicate_fieldname(e): @@ -308,18 +308,20 @@ class PostgresDatabase(Database): WHEN 'timestamp without time zone' THEN 'timestamp' ELSE a.data_type END AS type, - COUNT(b.indexdef) AS Index, + BOOL_OR(b.index) AS index, SPLIT_PART(COALESCE(a.column_default, NULL), '::', 1) AS default, BOOL_OR(b.unique) AS unique FROM information_schema.columns a LEFT JOIN - (SELECT indexdef, tablename, indexdef LIKE '%UNIQUE INDEX%' AS unique + (SELECT indexdef, tablename, + indexdef LIKE '%UNIQUE INDEX%' AS unique, + indexdef NOT LIKE '%UNIQUE INDEX%' AS index FROM pg_indexes WHERE tablename='{table_name}') b - ON SUBSTRING(b.indexdef, '\(.*\)') LIKE CONCAT('%', a.column_name, '%') + ON SUBSTRING(b.indexdef, '(.*)') LIKE CONCAT('%', a.column_name, '%') WHERE a.table_name = '{table_name}' - GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length;''' - .format(table_name=table_name), as_dict=1) + GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length; + '''.format(table_name=table_name), as_dict=1) def get_database_list(self, target): return [d[0] for d in self.sql("SELECT datname FROM pg_database;")] diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index 58153ca6ce..a2d5be0b70 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -11,8 +11,6 @@ class PostgresTable(DBTable): column_defs = self.get_column_definitions() if column_defs: add_text += ',\n'.join(column_defs) - # index - # index_defs = self.get_index_definitions() # TODO: set docstatus length # create table frappe.db.sql("""create table `%s` ( @@ -28,8 +26,25 @@ class PostgresTable(DBTable): idx bigint not null default '0', %s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text)) + self.create_indexes() frappe.db.commit() + def create_indexes(self): + create_index_query = "" + for key, col in self.columns.items(): + if (col.set_index + and col.fieldtype in frappe.db.type_map + and frappe.db.type_map.get(col.fieldtype)[0] + not in ('text', 'longtext')): + create_index_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, + table_name=self.table_name, + field=col.fieldname + ) + if create_index_query: + # nosemgrep + frappe.db.sql(create_index_query) + def alter(self): for col in self.columns.values(): col.build_for_alter_table(self.current_columns.get(col.fieldname.lower())) @@ -52,8 +67,8 @@ class PostgresTable(DBTable): query.append("ALTER COLUMN `{0}` TYPE {1} {2}".format( col.fieldname, get_definition(col.fieldtype, precision=col.precision, length=col.length), - using_clause) - ) + using_clause + )) for col in self.set_default: if col.fieldname=="name": @@ -73,37 +88,54 @@ class PostgresTable(DBTable): query.append("ALTER COLUMN `{}` SET DEFAULT {}".format(col.fieldname, col_default)) - create_index_query = "" + create_contraint_query = "" for col in self.add_index: # if index key not exists - create_index_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( + create_contraint_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( index_name=col.fieldname, table_name=self.table_name, field=col.fieldname) - drop_index_query = "" + for col in self.add_unique: + # if index key not exists + create_contraint_query += 'CREATE UNIQUE INDEX IF NOT EXISTS "unique_{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, + table_name=self.table_name, + field=col.fieldname + ) + + drop_contraint_query = "" for col in self.drop_index: # primary key if col.fieldname != 'name': # if index key exists - if not frappe.db.has_index(self.table_name, col.fieldname): - drop_index_query += 'DROP INDEX IF EXISTS "{}" ;'.format(col.fieldname) + drop_contraint_query += 'DROP INDEX IF EXISTS "{}" ;'.format(col.fieldname) - if query: - try: + for col in self.drop_unique: + # primary key + if col.fieldname != 'name': + # if index key exists + drop_contraint_query += 'DROP INDEX IF EXISTS "unique_{}" ;'.format(col.fieldname) + try: + if query: final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query)) - if final_alter_query: frappe.db.sql(final_alter_query) - if create_index_query: frappe.db.sql(create_index_query) - if drop_index_query: frappe.db.sql(drop_index_query) - except Exception as e: - # sanitize - if frappe.db.is_duplicate_fieldname(e): - frappe.throw(str(e)) - elif frappe.db.is_duplicate_entry(e): - fieldname = str(e).split("'")[-2] - frappe.throw(_("""{0} field cannot be set as unique in {1}, - as there are non-unique existing values""".format( - fieldname, self.table_name))) - raise e - else: - raise e + # nosemgrep + frappe.db.sql(final_alter_query) + if create_contraint_query: + # nosemgrep + frappe.db.sql(create_contraint_query) + if drop_contraint_query: + # nosemgrep + frappe.db.sql(drop_contraint_query) + except Exception as e: + # sanitize + if frappe.db.is_duplicate_fieldname(e): + frappe.throw(str(e)) + elif frappe.db.is_duplicate_entry(e): + fieldname = str(e).split("'")[-2] + frappe.throw( + _("{0} field cannot be set as unique in {1}, as there are non-unique existing values") + .format(fieldname, self.table_name) + ) + else: + raise e diff --git a/frappe/database/query.py b/frappe/database/query.py index 6d2be5fa25..587378b32f 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -308,7 +308,7 @@ class Permission: doctype = [doctype] for dt in doctype: - dt = re.sub("tab", "", dt) + dt = re.sub("^tab", "", dt) if not frappe.has_permission( dt, "select", diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 10582eff8f..9a6dd502dc 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -21,6 +21,7 @@ class DBTable: self.change_name = [] self.add_unique = [] self.add_index = [] + self.drop_unique = [] self.drop_index = [] self.set_default = [] @@ -219,8 +220,10 @@ class DbColumn: self.table.change_type.append(self) # unique - if((self.unique and not current_def['unique']) and column_type not in ('text', 'longtext')): + if ((self.unique and not current_def['unique']) and column_type not in ('text', 'longtext')): self.table.add_unique.append(self) + elif (current_def['unique'] and not self.unique) and column_type not in ('text', 'longtext'): + self.table.drop_unique.append(self) # default if (self.default_changed(current_def) @@ -230,9 +233,7 @@ class DbColumn: self.table.set_default.append(self) # index should be applied or dropped irrespective of type change - if ((current_def['index'] and not self.set_index and not self.unique) - or (current_def['unique'] and not self.unique)): - # to drop unique you have to drop index + if (current_def['index'] and not self.set_index) and column_type not in ('text', 'longtext'): self.table.drop_index.append(self) elif (not current_def['index'] and self.set_index) and not (column_type in ('text', 'longtext')): diff --git a/frappe/defaults.py b/frappe/defaults.py index eb98db449f..e249ef2099 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -126,7 +126,7 @@ def set_default(key, value, parent, parenttype="__default"): "defkey": key, "parent": parent }) - if value != None: + if value is not None: add_default(key, value, parent) else: _clear_cache(parent) @@ -187,7 +187,7 @@ def get_defaults_for(parent="__default"): """get all defaults""" defaults = frappe.cache().hget("defaults", parent) - if defaults==None: + if defaults is None: # sort descending because first default must get precedence table = DocType("DefaultValue") res = frappe.qb.from_(table).where( diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index e1789852f1..4164db679d 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -56,31 +56,6 @@ class Workspace: self.restricted_doctypes = frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache() self.restricted_pages = frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache() - def is_page_allowed(self): - cards = self.doc.get_link_groups() + get_custom_reports_and_doctypes(self.doc.module) - shortcuts = self.doc.shortcuts - - for section in cards: - links = loads(section.get('links')) if isinstance(section.get('links'), str) else section.get('links') - for item in links: - if self.is_item_allowed(item.get('link_to'), item.get('link_type')): - return True - - def _in_active_domains(item): - if not item.restrict_to_domain: - return True - else: - return item.restrict_to_domain in frappe.get_active_domains() - - for item in shortcuts: - if self.is_item_allowed(item.link_to, item.type) and _in_active_domains(item): - return True - - if not shortcuts and not self.doc.links: - return True - - return False - def is_permitted(self): """Returns true if Has Role is not set or the user is allowed.""" from frappe.utils import has_common @@ -346,20 +321,20 @@ def get_desktop_page(page): dict: dictionary of cards, charts and shortcuts to be displayed on website """ try: - wspace = Workspace(loads(page)) - wspace.build_workspace() + workspace = Workspace(loads(page)) + workspace.build_workspace() return { - 'charts': wspace.charts, - 'shortcuts': wspace.shortcuts, - 'cards': wspace.cards, - 'onboardings': wspace.onboardings + 'charts': workspace.charts, + 'shortcuts': workspace.shortcuts, + 'cards': workspace.cards, + 'onboardings': workspace.onboardings } except DoesNotExistError: frappe.log_error(frappe.get_traceback()) return {} @frappe.whitelist() -def get_wspace_sidebar_items(): +def get_workspace_sidebar_items(): """Get list of sidebar items for desk""" has_access = "Workspace Manager" in frappe.get_roles() @@ -385,8 +360,8 @@ def get_wspace_sidebar_items(): # Filter Page based on Permission for page in all_pages: try: - wspace = Workspace(page, True) - if wspace.is_permitted() and wspace.is_page_allowed() or has_access: + workspace = Workspace(page, True) + if has_access or workspace.is_permitted(): if page.public: pages.append(page) elif page.for_user == frappe.session.user: @@ -453,25 +428,24 @@ def get_custom_report_list(module): return out def save_new_widget(doc, page, blocks, new_widgets): + if loads(new_widgets): + widgets = _dict(loads(new_widgets)) - widgets = _dict(loads(new_widgets)) - - if widgets.chart: - doc.charts.extend(new_widget(widgets.chart, "Workspace Chart", "charts")) - if widgets.shortcut: - doc.shortcuts.extend(new_widget(widgets.shortcut, "Workspace Shortcut", "shortcuts")) - if widgets.card: - doc.build_links_table_from_card(widgets.card) + if widgets.chart: + doc.charts.extend(new_widget(widgets.chart, "Workspace Chart", "charts")) + if widgets.shortcut: + doc.shortcuts.extend(new_widget(widgets.shortcut, "Workspace Shortcut", "shortcuts")) + if widgets.card: + doc.build_links_table_from_card(widgets.card) # remove duplicate and unwanted widgets - if widgets: - clean_up(doc, blocks) + clean_up(doc, blocks) try: doc.save(ignore_permissions=True) except (ValidationError, TypeError) as e: # Create a json string to log - json_config = dumps(widgets, sort_keys=True, indent=4) + json_config = widgets and dumps(widgets, sort_keys=True, indent=4) # Error log body log = \ diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index a0523d90cd..20887f8886 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -42,13 +42,13 @@ def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None): doc = frappe.get_doc(doctype, d) try: message = '' - if action == 'submit' and doc.docstatus==0: + if action == 'submit' and doc.docstatus.is_draft(): doc.submit() message = _('Submiting {0}').format(doctype) - elif action == 'cancel' and doc.docstatus==1: + elif action == 'cancel' and doc.docstatus.is_submitted(): doc.cancel() message = _('Cancelling {0}').format(doctype) - elif action == 'update' and doc.docstatus < 2: + elif action == 'update' and not doc.docstatus.is_cancelled(): doc.update(data) doc.save() message = _('Updating {0}').format(doctype) diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index a49d5d5418..f0aa867c8a 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -52,3 +52,9 @@ def deferred_insert(routes): ] _deferred_insert("Route History", json.dumps(routes)) + +@frappe.whitelist() +def frequently_visited_links(): + return frappe.get_all('Route History', fields=['route', 'count(name) as count'], filters={ + 'user': frappe.session.user + }, group_by="route", order_by="count desc", limit=5) diff --git a/frappe/desk/doctype/workspace/workspace.json b/frappe/desk/doctype/workspace/workspace.json index 04975c69e3..211029dfcf 100644 --- a/frappe/desk/doctype/workspace/workspace.json +++ b/frappe/desk/doctype/workspace/workspace.json @@ -1,5 +1,6 @@ { "actions": [], + "allow_rename": 1, "autoname": "field:label", "beta": 1, "creation": "2020-01-23 13:45:59.470592", @@ -141,7 +142,7 @@ }, { "fieldname": "sequence_id", - "fieldtype": "Int", + "fieldtype": "Float", "label": "Sequence Id" }, { @@ -158,7 +159,7 @@ ], "in_create": 1, "links": [], - "modified": "2021-09-16 12:01:06.450622", + "modified": "2021-12-15 19:33:00.805265", "modified_by": "Administrator", "module": "Desk", "name": "Workspace", diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py index 94114e3918..b40f517350 100644 --- a/frappe/desk/doctype/workspace/workspace.py +++ b/frappe/desk/doctype/workspace/workspace.py @@ -6,6 +6,7 @@ import frappe from frappe import _ from frappe.modules.export_file import export_to_files from frappe.model.document import Document +from frappe.model.rename_doc import rename_doc from frappe.desk.desktop import save_new_widget from frappe.desk.utils import validate_route_conflict @@ -121,77 +122,157 @@ def get_report_type(report): report_type = frappe.get_value("Report", report, "report_type") return report_type in ["Query Report", "Script Report", "Custom Report"] +@frappe.whitelist() +def new_page(new_page): + if not loads(new_page): + return + + page = loads(new_page) + + if page.get("public") and not is_workspace_manager(): + return + + doc = frappe.new_doc('Workspace') + doc.title = page.get('title') + doc.icon = page.get('icon') + doc.content = page.get('content') + doc.parent_page = page.get('parent_page') + doc.label = page.get('label') + doc.for_user = page.get('for_user') + doc.public = page.get('public') + doc.sequence_id = last_sequence_id(doc) + 1 + doc.save(ignore_permissions=True) + + return doc @frappe.whitelist() -def save_page(title, icon, parent, public, sb_public_items, sb_private_items, deleted_pages, new_widgets, blocks, save): - save = frappe.parse_json(save) +def save_page(title, public, new_widgets, blocks): public = frappe.parse_json(public) - if save: - doc = frappe.new_doc('Workspace') - doc.title = title - doc.icon = icon - doc.content = blocks - doc.parent_page = parent - if public: - doc.label = title - doc.public = 1 - else: - doc.label = title + "-" + frappe.session.user - doc.for_user = frappe.session.user - doc.save(ignore_permissions=True) - else: - if public: - filters = { - 'public': public, - 'label': title - } - else: - filters = { - 'for_user': frappe.session.user, - 'label': title + "-" + frappe.session.user - } - pages = frappe.get_list("Workspace", filters=filters) - if pages: - doc = frappe.get_doc("Workspace", pages[0]) + filters = { + 'public': public, + 'label': title + } - doc.content = blocks - doc.save(ignore_permissions=True) + if not public: + filters = { + 'for_user': frappe.session.user, + 'label': title + "-" + frappe.session.user + } + pages = frappe.get_list("Workspace", filters=filters) + if pages: + doc = frappe.get_doc("Workspace", pages[0]) - if loads(new_widgets): - save_new_widget(doc, title, blocks, new_widgets) + doc.content = blocks + doc.save(ignore_permissions=True) - if loads(sb_public_items) or loads(sb_private_items): - sort_pages(loads(sb_public_items), loads(sb_private_items)) - - if loads(deleted_pages): - return delete_pages(loads(deleted_pages)) + save_new_widget(doc, title, blocks, new_widgets) return {"name": title, "public": public, "label": doc.label} -def delete_pages(deleted_pages): - for page in deleted_pages: - if page.get("public") and not is_workspace_manager(): - return {"name": page.get("title"), "public": 1, "label": page.get("label")} +@frappe.whitelist() +def update_page(name, title, icon, parent, public): + public = frappe.parse_json(public) - if frappe.db.exists("Workspace", page.get("name")): - frappe.get_doc("Workspace", page.get("name")).delete(ignore_permissions=True) + doc = frappe.get_doc("Workspace", name) - return {"name": "Home", "public": 1, "label": "Home"} + filters = { + 'parent_page': doc.title, + 'public': doc.public + } + child_docs = frappe.get_list("Workspace", filters=filters) + if doc: + doc.title = title + doc.icon = icon + doc.parent_page = parent + if doc.public != public: + doc.sequence_id = frappe.db.count('Workspace', {'public':public}, cache=True) + doc.public = public + doc.for_user = '' if public else doc.for_user or frappe.session.user + doc.label = '{0}-{1}'.format(title, doc.for_user) if doc.for_user else title + doc.save(ignore_permissions=True) + + if name != doc.label: + rename_doc("Workspace", name, doc.label, force=True, ignore_permissions=True) + + # update new name and public in child pages + if child_docs: + for child in child_docs: + child_doc = frappe.get_doc("Workspace", child.name) + child_doc.parent_page = doc.title + child_doc.public = doc.public + child_doc.save(ignore_permissions=True) + + return {"name": doc.title, "public": doc.public, "label": doc.label} + +@frappe.whitelist() +def duplicate_page(page_name, new_page): + if not loads(new_page): + return + + new_page = loads(new_page) + + if new_page.get("is_public") and not is_workspace_manager(): + return + + old_doc = frappe.get_doc("Workspace", page_name) + doc = frappe.copy_doc(old_doc) + doc.title = new_page.get('title') + doc.icon = new_page.get('icon') + doc.parent_page = new_page.get('parent') or '' + doc.public = new_page.get('is_public') + doc.for_user = '' + doc.label = doc.title + if not doc.public: + doc.for_user = doc.for_user or frappe.session.user + doc.label = '{0}-{1}'.format(doc.title, doc.for_user) + doc.name = doc.label + if old_doc.public == doc.public: + doc.sequence_id += 0.1 + else: + doc.sequence_id = last_sequence_id(doc) + 1 + doc.insert(ignore_permissions=True) + + return doc + +@frappe.whitelist() +def delete_page(page): + if not loads(page): + return + + page = loads(page) + + if page.get("public") and not is_workspace_manager(): + return + + if frappe.db.exists("Workspace", page.get("name")): + frappe.get_doc("Workspace", page.get("name")).delete(ignore_permissions=True) + + return {"name": page.get("name"), "public": page.get("public"), "title": page.get("title")} + +@frappe.whitelist() def sort_pages(sb_public_items, sb_private_items): - wspace_public_pages = get_page_list(['name', 'title'], {'public': 1}) - wspace_private_pages = get_page_list(['name', 'title'], {'for_user': frappe.session.user}) + if not loads(sb_public_items) and not loads(sb_private_items): + return + + sb_public_items = loads(sb_public_items) + sb_private_items = loads(sb_private_items) + + workspace_public_pages = get_page_list(['name', 'title'], {'public': 1}) + workspace_private_pages = get_page_list(['name', 'title'], {'for_user': frappe.session.user}) if sb_private_items: - sort_page(wspace_private_pages, sb_private_items) + return sort_page(workspace_private_pages, sb_private_items) if sb_public_items and is_workspace_manager(): - sort_page(wspace_public_pages, sb_public_items) + return sort_page(workspace_public_pages, sb_public_items) -def sort_page(wspace_pages, pages): + return False + +def sort_page(workspace_pages, pages): for seq, d in enumerate(pages): - for page in wspace_pages: + for page in workspace_pages: if page.title == d.get('title'): doc = frappe.get_doc('Workspace', page.name) doc.sequence_id = seq + 1 @@ -199,6 +280,27 @@ def sort_page(wspace_pages, pages): doc.save(ignore_permissions=True) break + return True + +def last_sequence_id(doc): + doc_exists = frappe.db.exists({ + 'doctype': 'Workspace', + 'public': doc.public, + 'for_user': doc.for_user + }) + + if not doc_exists: + return 0 + + return frappe.db.get_list('Workspace', + fields=['sequence_id'], + filters={ + 'public': doc.public, + 'for_user': doc.for_user + }, + order_by="sequence_id desc" + )[0].sequence_id + def get_page_list(fields, filters): return frappe.get_list("Workspace", fields=fields, filters=filters, order_by='sequence_id asc') diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 0e644c3cf5..58d5b30103 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -91,33 +91,82 @@ def get_docinfo(doc=None, doctype=None, name=None): raise frappe.PermissionError all_communications = _get_communications(doc.doctype, doc.name) - automated_messages = filter(lambda x: x['communication_type'] == 'Automated Message', all_communications) - communications_except_auto_messages = filter(lambda x: x['communication_type'] != 'Automated Message', all_communications) + automated_messages = [msg for msg in all_communications if msg['communication_type'] == 'Automated Message'] + communications_except_auto_messages = [msg for msg in all_communications if msg['communication_type'] != 'Automated Message'] - frappe.response["docinfo"] = { + docinfo = frappe._dict(user_info = {}) + + add_comments(doc, docinfo) + + docinfo.update({ "attachments": get_attachments(doc.doctype, doc.name), - "attachment_logs": get_comments(doc.doctype, doc.name, 'attachment'), "communications": communications_except_auto_messages, "automated_messages": automated_messages, - 'comments': get_comments(doc.doctype, doc.name), 'total_comments': len(json.loads(doc.get('_comments') or '[]')), 'versions': get_versions(doc), "assignments": get_assignments(doc.doctype, doc.name), - "assignment_logs": get_comments(doc.doctype, doc.name, 'assignment'), "permissions": get_doc_permissions(doc), "shared": frappe.share.get_users(doc.doctype, doc.name), - "info_logs": get_comments(doc.doctype, doc.name, comment_type=['Info', 'Edit', 'Label']), - "share_logs": get_comments(doc.doctype, doc.name, 'share'), - "like_logs": get_comments(doc.doctype, doc.name, 'Like'), - "workflow_logs": get_comments(doc.doctype, doc.name, comment_type="Workflow"), "views": get_view_logs(doc.doctype, doc.name), "energy_point_logs": get_point_logs(doc.doctype, doc.name), "additional_timeline_content": get_additional_timeline_content(doc.doctype, doc.name), "milestones": get_milestones(doc.doctype, doc.name), "is_document_followed": is_document_followed(doc.doctype, doc.name, frappe.session.user), "tags": get_tags(doc.doctype, doc.name), - "document_email": get_document_email(doc.doctype, doc.name) - } + "document_email": get_document_email(doc.doctype, doc.name), + }) + + update_user_info(docinfo) + + frappe.response["docinfo"] = docinfo + return docinfo + +def add_comments(doc, docinfo): + # divide comments into separate lists + docinfo.comments = [] + docinfo.shared = [] + docinfo.assignment_logs = [] + docinfo.attachment_logs = [] + docinfo.info_logs = [] + docinfo.like_logs = [] + docinfo.workflow_logs = [] + + comments = frappe.get_all("Comment", + fields=["name", "creation", "content", "owner", "comment_type"], + filters={ + "reference_doctype": doc.doctype, + "reference_name": doc.name + } + ) + + for c in comments: + if c.comment_type == "Comment": + c.content = frappe.utils.markdown(c.content) + docinfo.comments.append(c) + + elif c.comment_type in ('Shared', 'Unshared'): + docinfo.shared.append(c) + + elif c.comment_type in ('Assignment Completed', 'Assigned'): + docinfo.assignment_logs.append(c) + + elif c.comment_type in ('Attachment', 'Attachment Removed'): + docinfo.attachment_logs.append(c) + + elif c.comment_type in ('Info', 'Edit', 'Label'): + docinfo.info_logs.append(c) + + elif c.comment_type == "Like": + docinfo.like_logs.append(c) + + elif c.comment_type == "Workflow": + docinfo.workflow_logs.append(c) + + frappe.utils.add_user_info(c.owner, docinfo.user_info) + + + return comments + def get_milestones(doctype, name): return frappe.db.get_all('Milestone', fields = ['creation', 'owner', 'track_field', 'value'], @@ -252,7 +301,7 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields= return communications def get_assignments(dt, dn): - cl = frappe.get_all("ToDo", + return frappe.get_all("ToDo", fields=['name', 'allocated_to as owner', 'description', 'status'], filters={ 'reference_type': dt, @@ -260,8 +309,6 @@ def get_assignments(dt, dn): 'status': ('!=', 'Cancelled'), }) - return cl - @frappe.whitelist() def get_badge_info(doctypes, filters): filters = json.loads(filters) @@ -319,3 +366,24 @@ def get_additional_timeline_content(doctype, docname): contents.extend(frappe.get_attr(method)(doctype, docname) or []) return contents + +def update_user_info(docinfo): + for d in docinfo.communications: + frappe.utils.add_user_info(d.sender, docinfo.user_info) + + for d in docinfo.shared: + frappe.utils.add_user_info(d.user, docinfo.user_info) + + for d in docinfo.assignments: + frappe.utils.add_user_info(d.owner, docinfo.user_info) + + for d in docinfo.views: + frappe.utils.add_user_info(d.owner, docinfo.user_info) + +@frappe.whitelist() +def get_user_info_for_viewers(users): + user_info = {} + for user in json.loads(users): + frappe.utils.add_user_info(user, user_info) + + return user_info diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py index e2e2c4c155..7a9c211c3c 100644 --- a/frappe/desk/moduleview.py +++ b/frappe/desk/moduleview.py @@ -524,7 +524,7 @@ def get_last_modified(doctype): raise # hack: save as -1 so that it is cached - if last_modified==None: + if last_modified is None: last_modified = -1 return last_modified diff --git a/frappe/desk/page/user_profile/user_profile.py b/frappe/desk/page/user_profile/user_profile.py index 73df6d78cb..0d91fd0d91 100644 --- a/frappe/desk/page/user_profile/user_profile.py +++ b/frappe/desk/page/user_profile/user_profile.py @@ -30,7 +30,7 @@ def get_energy_points_percentage_chart_data(user, field): as_list = True) return { - "labels": [r[0] for r in result if r[0] != None], + "labels": [r[0] for r in result if r[0] is not None], "datasets": [{ "values": [r[1] for r in result] }] diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index fb150e4bea..4001d0b9cf 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -12,14 +12,14 @@ from io import StringIO from frappe.core.doctype.access_log.access_log import make_access_log from frappe.utils import cstr, format_duration from frappe.model.base_document import get_controller - +from frappe.utils import add_user_info @frappe.whitelist() @frappe.read_only() def get(): args = get_form_params() # If virtual doctype get data from controller het_list method - if frappe.db.get_value("DocType", filters={"name": args.doctype}, fieldname="is_virtual"): + if is_virtual_doctype(args.doctype): controller = get_controller(args.doctype) data = compress(controller(args.doctype).get_list(args)) else: @@ -29,17 +29,31 @@ def get(): @frappe.whitelist() @frappe.read_only() def get_list(): - # uncompressed (refactored from frappe.model.db_query.get_list) - return execute(**get_form_params()) + args = get_form_params() + + if is_virtual_doctype(args.doctype): + controller = get_controller(args.doctype) + data = controller(args.doctype).get_list(args) + else: + # uncompressed (refactored from frappe.model.db_query.get_list) + data = execute(**args) + + return data @frappe.whitelist() @frappe.read_only() def get_count(): args = get_form_params() - distinct = 'distinct ' if args.distinct=='true' else '' - args.fields = [f"count({distinct}`tab{args.doctype}`.name) as total_count"] - return execute(**args)[0].get('total_count') + if is_virtual_doctype(args.doctype): + controller = get_controller(args.doctype) + data = controller(args.doctype).get_count(args) + else: + distinct = 'distinct ' if args.distinct=='true' else '' + args.fields = [f"count({distinct}`tab{args.doctype}`.name) as total_count"] + data = execute(**args)[0].get('total_count') + + return data def execute(doctype, *args, **kwargs): return DatabaseQuery(doctype).execute(*args, **kwargs) @@ -219,6 +233,8 @@ def compress(data, args=None): """separate keys and values""" from frappe.desk.query_report import add_total_row + user_info = {} + if not data: return data if args is None: args = {} @@ -230,13 +246,19 @@ def compress(data, args=None): new_row.append(row.get(key)) values.append(new_row) + # add user info for assignments (avatar) + if row._assign: + for user in json.loads(row._assign): + add_user_info(user, user_info) + if args.get("add_total_row"): meta = frappe.get_meta(args.doctype) values = add_total_row(values, keys, meta) return { "keys": keys, - "values": values + "values": values, + "user_info": user_info } @frappe.whitelist() @@ -297,7 +319,7 @@ def export_query(): if add_totals_row: ret = append_totals_row(ret) - data = [['Sr'] + get_labels(db_query.fields, doctype)] + data = [[_('Sr')] + get_labels(db_query.fields, doctype)] for i, row in enumerate(ret): data.append([i+1] + list(row)) @@ -356,7 +378,8 @@ def get_labels(fields, doctype): for key in fields: key = key.split(" as ")[0] - if key.startswith(('count(', 'sum(', 'avg(')): continue + if key.startswith(('count(', 'sum(', 'avg(')): + continue if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") @@ -364,10 +387,16 @@ def get_labels(fields, doctype): parenttype = doctype fieldname = fieldname.strip("`") - df = frappe.get_meta(parenttype).get_field(fieldname) - label = df.label if df else fieldname.title() - if label in labels: - label = doctype + ": " + label + if parenttype == doctype and fieldname == "name": + label = _("ID", context="Label of name column in report") + else: + df = frappe.get_meta(parenttype).get_field(fieldname) + label = _(df.label if df else fieldname.title()) + if parenttype != doctype: + # If the column is from a child table, append the child doctype. + # For example, "Item Code (Sales Invoice Item)". + label += f" ({ _(parenttype) })" + labels.append(label) return labels @@ -430,7 +459,14 @@ def get_sidebar_stats(stats, doctype, filters=None): if filters is None: filters = [] - return {"stats": get_stats(stats, doctype, filters)} + if is_virtual_doctype(doctype): + controller = get_controller(doctype) + args = {"stats": stats, "filters": filters} + data = controller(doctype).get_stats(args) + else: + data = get_stats(stats, doctype, filters) + + return {"stats": data} @frappe.whitelist() @frappe.read_only() @@ -552,7 +588,7 @@ def get_match_cond(doctype, as_condition=True): return ((' and ' + cond) if cond else "").replace("%", "%%") def build_match_conditions(doctype, user=None, as_condition=True): - match_conditions = DatabaseQuery(doctype, user=user).build_match_conditions(as_condition=as_condition) + match_conditions = DatabaseQuery(doctype, user=user).build_match_conditions(as_condition=as_condition) if as_condition: return match_conditions.replace("%", "%%") else: @@ -590,3 +626,7 @@ def get_filters_cond(doctype, filters, conditions, ignore_permissions=None, with else: cond = '' return cond + +def is_virtual_doctype(doctype): + return frappe.db.get_value("DocType", doctype, "is_virtual") + diff --git a/frappe/desk/search.py b/frappe/desk/search.py index db88e6ec52..95397070ae 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -107,7 +107,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, else: filters.append([doctype, f[0], "=", f[1]]) - if filters==None: + if filters is None: filters = [] or_filters = [] diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 3f8d399c52..a05e20da24 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -1,5 +1,6 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE + import email.utils import functools import imaplib @@ -7,6 +8,7 @@ import socket import time from datetime import datetime, timedelta from poplib import error_proto +from typing import List import frappe from frappe import _, are_emails_muted, safe_encode @@ -82,9 +84,6 @@ class EmailAccount(Document): if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - #if self.enable_incoming and not self.append_to: - # frappe.throw(_("Append To is mandatory for incoming mails")) - if (not self.awaiting_password and not frappe.local.flags.in_install and not frappe.local.flags.in_patch): if self.password or self.smtp_server in ('127.0.0.1', 'localhost'): @@ -442,7 +441,7 @@ class EmailAccount(Document): frappe.db.rollback() except Exception: frappe.db.rollback() - frappe.log_error('email_account.receive') + frappe.log_error(title="EmailAccount.receive") if self.use_imap: self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback()) exceptions.append(frappe.get_traceback()) @@ -458,7 +457,7 @@ class EmailAccount(Document): if exceptions: raise Exception(frappe.as_json(exceptions)) - def get_inbound_mails(self, test_mails=None): + def get_inbound_mails(self, test_mails=None) -> List[InboundMail]: """retrive and return inbound mails. """ @@ -625,7 +624,6 @@ class EmailAccount(Document): if frappe.db.exists("Email Account", {"enable_automatic_linking": 1, "name": ('!=', self.name)}): frappe.throw(_("Automatic Linking can be activated only for one Email Account.")) - def append_email_to_sent_folder(self, message): email_server = None try: @@ -643,7 +641,8 @@ class EmailAccount(Document): message = safe_encode(message) email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message) except Exception: - frappe.log_error() + frappe.log_error(title="EmailAccount.append_email_to_sent_folder") + @frappe.whitelist() def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None): diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 9730004065..4da83bd0d2 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -11,7 +11,6 @@ import quopri from email.parser import Parser from email.policy import SMTPUTF8 from html2text import html2text -from six.moves import html_parser as HTMLParser import frappe from frappe import _, safe_encode, task @@ -20,6 +19,7 @@ from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message from frappe.email.email_body import add_attachment, get_formatted_html, get_email from frappe.utils import cint, split_emails, add_days, nowdate, cstr, get_hook_method from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.query_builder.utils import DocType MAX_RETRY_COUNT = 3 @@ -444,7 +444,7 @@ class QueueBuilder: try: text_content = html2text(self._message) - except HTMLParser.HTMLParseError: + except Exception: text_content = "See html attachment" return text_content + unsubscribe_text_message @@ -477,18 +477,24 @@ class QueueBuilder: all_ids = list(set(self.recipients + self.cc)) - EmailUnsubscribe = frappe.qb.DocType("Email Unsubscribe") + EmailUnsubscribe = DocType("Email Unsubscribe") + + unsubscribed = ( + frappe.qb.from_(EmailUnsubscribe).select( + EmailUnsubscribe.email + ).where( + EmailUnsubscribe.email.isin(all_ids) + & ( + ( + (EmailUnsubscribe.reference_doctype == self.reference_doctype) + & (EmailUnsubscribe.reference_name == self.reference_name) + ) | ( + EmailUnsubscribe.global_unsubscribe == 1 + ) + ) + ).distinct() + ).run(pluck=True) - unsubscribed = (frappe.qb.from_(EmailUnsubscribe) - .select(EmailUnsubscribe.email) - .where(EmailUnsubscribe.email.isin(all_ids) & - ( - ( - (EmailUnsubscribe.reference_doctype == self.reference_doctype) & (EmailUnsubscribe.reference_name == self.reference_name) - ) | EmailUnsubscribe.global_unsubscribe == 1 - ) - ).distinct() - ).run(pluck=True) self._unsubscribed_user_emails = unsubscribed or [] return self._unsubscribed_user_emails diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 77979f9735..3fd96bdb6b 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -137,7 +137,7 @@ def get_context(context): if self.set_property_after_alert: allow_update = True - if doc.docstatus == 1 and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit: + if doc.docstatus.is_submitted() and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit: allow_update = False try: if allow_update and not doc.flags.in_notification_update: diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 8145091f69..098a2e6c80 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -104,6 +104,7 @@ class DocumentAlreadyRestored(ValidationError): pass class AttachmentLimitReached(ValidationError): pass class QueryTimeoutError(Exception): pass class QueryDeadlockError(Exception): pass +class TooManyWritesError(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass diff --git a/frappe/installer.py b/frappe/installer.py old mode 100755 new mode 100644 index b50fa4a3b5..d892ff4ddc --- a/frappe/installer.py +++ b/frappe/installer.py @@ -154,7 +154,7 @@ def install_app(name, verbose=False, set_as_patched=True): for before_install in app_hooks.before_install or []: out = frappe.get_attr(before_install)() - if out==False: + if out is False: return if name != "frappe": @@ -346,14 +346,15 @@ def post_install(rebuild_website=False): def set_all_patches_as_completed(app): - patch_path = os.path.join(frappe.get_pymodule_path(app), "patches.txt") - if os.path.exists(patch_path): - for patch in frappe.get_file_items(patch_path): - frappe.get_doc({ - "doctype": "Patch Log", - "patch": patch - }).insert(ignore_permissions=True) - frappe.db.commit() + from frappe.modules.patch_handler import get_patches_from_app + + patches = get_patches_from_app(app) + for patch in patches: + frappe.get_doc({ + "doctype": "Patch Log", + "patch": patch + }).insert(ignore_permissions=True) + frappe.db.commit() def init_singles(): diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 416d656d90..4242676d94 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -65,10 +65,7 @@ def get_latest_backup_file(with_files=False): return database, config -def get_file_size(file_path, unit): - if not unit: - unit = "MB" - +def get_file_size(file_path, unit='MB'): file_size = os.path.getsize(file_path) memory_size_unit_mapper = {"KB": 1, "MB": 2, "GB": 3, "TB": 4} @@ -99,7 +96,7 @@ def get_chunk_site(file_size): def validate_file_size(): frappe.flags.create_new_backup = True latest_file, site_config = get_latest_backup_file() - file_size = get_file_size(latest_file, unit="GB") + file_size = get_file_size(latest_file, unit="GB") if latest_file else 0 if file_size > 1: frappe.flags.create_new_backup = False diff --git a/frappe/integrations/workspace/integrations/integrations.json b/frappe/integrations/workspace/integrations/integrations.json index b85056e3ef..bbd2e1199f 100644 --- a/frappe/integrations/workspace/integrations/integrations.json +++ b/frappe/integrations/workspace/integrations/integrations.json @@ -1,6 +1,6 @@ { "charts": [], - "content": "[{\"type\": \"header\", \"data\": {\"text\": \"Reports & Masters\", \"level\": 4, \"col\": 12}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Backup\", \"col\": 4}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Google Services\", \"col\": 4}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Authentication\", \"col\": 4}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Payments\", \"col\": 4}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Settings\", \"col\": 4}}]", + "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Backup\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Google Services\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Authentication\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Payments\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Settings\",\"col\":4}}]", "creation": "2020-03-02 15:16:18.714190", "docstatus": 0, "doctype": "Workspace", @@ -260,7 +260,7 @@ "type": "Link" } ], - "modified": "2021-08-05 12:16:00.355268", + "modified": "2022-01-13 17:39:01.292154", "modified_by": "Administrator", "module": "Integrations", "name": "Integrations", @@ -269,7 +269,7 @@ "public": 1, "restrict_to_domain": "", "roles": [], - "sequence_id": 15, + "sequence_id": 15.0, "shortcuts": [], "title": "Integrations" } \ No newline at end of file diff --git a/frappe/migrate.py b/frappe/migrate.py index 6abc38796f..d13fe858f7 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -19,6 +19,8 @@ from frappe.modules.utils import sync_customizations from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs from frappe.search.website_search import build_index_for_all_routes from frappe.database.schema import add_column +from frappe.modules.patch_handler import PatchType + def migrate(verbose=True, skip_failing=False, skip_search_index=False): @@ -59,16 +61,13 @@ Otherwise, check the server logs and ensure that all the required services are r clear_global_cache() - #run before_migrate hooks for app in frappe.get_installed_apps(): for fn in frappe.get_hooks('before_migrate', app_name=app): frappe.get_attr(fn)() - # run patches - frappe.modules.patch_handler.run_all(skip_failing) - - # sync + frappe.modules.patch_handler.run_all(skip_failing=skip_failing, patch_type=PatchType.pre_model_sync) frappe.model.sync.sync_all() + frappe.modules.patch_handler.run_all(skip_failing=skip_failing, patch_type=PatchType.post_model_sync) frappe.translate.clear_cache() sync_jobs() sync_fixtures() @@ -78,18 +77,16 @@ Otherwise, check the server logs and ensure that all the required services are r frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu() - # syncs statics + # syncs static files clear_website_cache() # updating installed applications data frappe.get_single('Installed Applications').update_versions() - #run after_migrate hooks for app in frappe.get_installed_apps(): for fn in frappe.get_hooks('after_migrate', app_name=app): frappe.get_attr(fn)() - # build web_routes index if not skip_search_index: # Run this last as it updates the current session print('Building search index for {}'.format(frappe.local.site)) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 26a4658c36..94f2c5ea18 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -1,5 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE + import frappe import datetime from frappe import _ @@ -11,6 +12,7 @@ from frappe.model import display_fieldtypes from frappe.utils import (cint, flt, now, cstr, strip_html, sanitize_html, sanitize_email, cast_fieldtype) from frappe.utils.html_utils import unescape_html +from frappe.model.docstatus import DocStatus max_positive_value = { 'smallint': 2 ** 15, @@ -20,6 +22,7 @@ max_positive_value = { DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link') + def get_controller(doctype): """Returns the **class** object of the given DocType. For `custom` type, returns `frappe.model.document.Document`. @@ -172,7 +175,7 @@ class BaseDocument(object): ... }) """ - if value==None: + if value is None: value={} if isinstance(value, (dict, BaseDocument)): if not self.__dict__.get(key): @@ -224,7 +227,7 @@ class BaseDocument(object): value.parentfield = key if value.docstatus is None: - value.docstatus = 0 + value.docstatus = DocStatus.draft() if not getattr(value, "idx", None): value.idx = len(self.get(key) or []) + 1 @@ -272,7 +275,7 @@ class BaseDocument(object): )): d[fieldname] = str(d[fieldname]) - if d[fieldname] == None and ignore_nulls: + if d[fieldname] is None and ignore_nulls: del d[fieldname] return d @@ -282,8 +285,11 @@ class BaseDocument(object): if key not in self.__dict__: self.__dict__[key] = None - if key in ("idx", "docstatus") and self.__dict__[key] is None: - self.__dict__[key] = 0 + if self.__dict__[key] is None: + if key == "docstatus": + self.docstatus = DocStatus.draft() + elif key == "idx": + self.__dict__[key] = 0 for key in self.get_valid_columns(): if key not in self.__dict__: @@ -304,6 +310,14 @@ class BaseDocument(object): def is_new(self): return self.get("__islocal") + @property + def docstatus(self): + return DocStatus(self.get("docstatus")) + + @docstatus.setter + def docstatus(self, value): + self.__dict__["docstatus"] = DocStatus(cint(value)) + def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False): doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str) doc["doctype"] = self.doctype @@ -492,7 +506,7 @@ class BaseDocument(object): self.set(df.fieldname, flt(self.get(df.fieldname))) if self.docstatus is not None: - self.docstatus = cint(self.docstatus) + self.docstatus = DocStatus(cint(self.docstatus)) def _get_missing_mandatory_fields(self): """Get mandatory fields that do not have any values""" @@ -581,7 +595,7 @@ class BaseDocument(object): setattr(self, df.fieldname, values.name) for _df in fields_to_fetch: - if self.is_new() or self.docstatus != 1 or _df.allow_on_submit: + if self.is_new() or not self.docstatus.is_submitted() or _df.allow_on_submit: self.set_fetch_from_value(doctype, _df, values) notify_link_count(doctype, docname) @@ -591,7 +605,7 @@ class BaseDocument(object): elif (df.fieldname != "amended_from" and (is_submittable or self.meta.is_submittable) and frappe.get_meta(doctype).is_submittable - and cint(frappe.db.get_value(doctype, docname, "docstatus"))==2): + and cint(frappe.db.get_value(doctype, docname, "docstatus")) == DocStatus.cancelled()): cancelled_links.append((df.fieldname, docname, get_msg(df, docname))) @@ -646,8 +660,6 @@ class BaseDocument(object): value, comma_options)) def _validate_data_fields(self): - from frappe.core.doctype.user.user import STANDARD_USERS - # data_field options defined in frappe.model.data_field_options for data_field in self.meta.get_data_fields(): data = self.get(data_field.fieldname) @@ -658,7 +670,7 @@ class BaseDocument(object): continue if data_field_options == "Email": - if (self.owner in STANDARD_USERS) and (data in STANDARD_USERS): + if (self.owner in frappe.STANDARD_USERS) and (data in frappe.STANDARD_USERS): continue for email_address in frappe.utils.split_emails(data): frappe.utils.validate_email_address(email_address, throw=True) @@ -807,8 +819,8 @@ class BaseDocument(object): or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode", "Code") # cancelled and submit but not update after submit should be ignored - or self.docstatus==2 - or (self.docstatus==1 and not df.get("allow_on_submit"))): + or self.docstatus.is_cancelled() + or (self.docstatus.is_submitted() and not df.get("allow_on_submit"))): continue else: diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 51d53c69a5..79be261981 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -545,7 +545,7 @@ class DatabaseQuery(object): elif f.operator.lower() in ("like", "not like") or (isinstance(f.value, str) and (not df or df.fieldtype not in ["Float", "Int", "Currency", "Percent", "Check"])): - value = "" if f.value==None else f.value + value = "" if f.value is None else f.value fallback = "''" if f.operator.lower() in ("like", "not like") and isinstance(value, str): diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 2fddcf9e33..afe01d9106 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -212,7 +212,7 @@ def check_permission_and_not_submitted(doc): .format(doc.doctype, doc.name), raise_exception=frappe.PermissionError) # check if submitted - if doc.docstatus == 1: + if doc.docstatus.is_submitted(): frappe.msgprint(_("{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.").format(_(doc.doctype), doc.name, "", ""), raise_exception=True) diff --git a/frappe/model/docstatus.py b/frappe/model/docstatus.py new file mode 100644 index 0000000000..01aab1e491 --- /dev/null +++ b/frappe/model/docstatus.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. See LICENSE + + +class DocStatus(int): + def is_draft(self): + return self == self.draft() + + def is_submitted(self): + return self == self.submitted() + + def is_cancelled(self): + return self == self.cancelled() + + @classmethod + def draft(cls): + return cls(0) + + @classmethod + def submitted(cls): + return cls(1) + + @classmethod + def cancelled(cls): + return cls(2) diff --git a/frappe/model/document.py b/frappe/model/document.py index 6b27049d08..b686135d15 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,13 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe +import hashlib +import json import time +from werkzeug.exceptions import NotFound + +import frappe from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc -from werkzeug.exceptions import NotFound, Forbidden -import hashlib, json +from frappe.model.naming import set_new_name +from frappe.model.docstatus import DocStatus from frappe.model import optional_fields, table_fields from frappe.model.workflow import validate_workflow from frappe.model.workflow import set_workflow_state_on_action @@ -17,6 +20,7 @@ from frappe.desk.form.document_follow import follow_document from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event from frappe.utils.data import get_absolute_url + # once_only validation # methods @@ -188,6 +192,8 @@ class Document(BaseDocument): is not set. :param permtype: one of `read`, `write`, `submit`, `cancel`, `delete`""" + import frappe.permissions + if self.flags.ignore_permissions: return True return frappe.permissions.has_permission(self.doctype, permtype, self, verbose=verbose) @@ -209,13 +215,13 @@ class Document(BaseDocument): self.flags.notifications_executed = [] - if ignore_permissions!=None: + if ignore_permissions is not None: self.flags.ignore_permissions = ignore_permissions - if ignore_links!=None: + if ignore_links is not None: self.flags.ignore_links = ignore_links - if ignore_mandatory!=None: + if ignore_mandatory is not None: self.flags.ignore_mandatory = ignore_mandatory self.set("__islocal", True) @@ -295,7 +301,7 @@ class Document(BaseDocument): self.flags.notifications_executed = [] - if ignore_permissions!=None: + if ignore_permissions is not None: self.flags.ignore_permissions = ignore_permissions self.flags.ignore_version = frappe.flags.in_test if ignore_version is None else ignore_version @@ -305,9 +311,6 @@ class Document(BaseDocument): self.check_permission("write", "save") - if self.docstatus == 2: - self._rename_doc_on_cancel() - self.set_user_and_timestamp() self.set_docstatus() self.check_if_latest() @@ -439,7 +442,7 @@ class Document(BaseDocument): values = self.as_dict() # format values for key, value in values.items(): - if value==None: + if value is None: values[key] = "" return values @@ -472,7 +475,7 @@ class Document(BaseDocument): # We'd probably want the creation and owner to be set via API # or Data import at some point, that'd have to be handled here - if self.is_new(): + if self.is_new() and not (frappe.flags.in_patch or frappe.flags.in_migrate): self.creation = self.modified self.owner = self.modified_by @@ -487,8 +490,8 @@ class Document(BaseDocument): frappe.flags.currently_saving.append((self.doctype, self.name)) def set_docstatus(self): - if self.docstatus==None: - self.docstatus=0 + if self.docstatus is None: + self.docstatus = DocStatus.draft() for d in self.get_all_children(): d.docstatus = self.docstatus @@ -718,6 +721,7 @@ class Document(BaseDocument): else: tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` where name = %s for update""".format(self.doctype), self.name, as_dict=True) + if not tmp: frappe.throw(_("Record does not exist")) else: @@ -738,7 +742,7 @@ class Document(BaseDocument): else: self.check_docstatus_transition(0) - def check_docstatus_transition(self, docstatus): + def check_docstatus_transition(self, to_docstatus): """Ensures valid `docstatus` transition. Valid transitions are (number in brackets is `docstatus`): @@ -749,31 +753,32 @@ class Document(BaseDocument): """ if not self.docstatus: - self.docstatus = 0 - if docstatus==0: - if self.docstatus==0: + self.docstatus = DocStatus.draft() + + if to_docstatus == DocStatus.draft(): + if self.docstatus.is_draft(): self._action = "save" - elif self.docstatus==1: + elif self.docstatus.is_submitted(): self._action = "submit" self.check_permission("submit") - elif self.docstatus==2: + elif self.docstatus.is_cancelled(): raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)")) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) - elif docstatus==1: - if self.docstatus==1: + elif to_docstatus == DocStatus.submitted(): + if self.docstatus.is_submitted(): self._action = "update_after_submit" self.check_permission("submit") - elif self.docstatus==2: + elif self.docstatus.is_cancelled(): self._action = "cancel" self.check_permission("cancel") - elif self.docstatus==0: + elif self.docstatus.is_draft(): raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 (Submitted) to 0 (Draft)")) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) - elif docstatus==2: + elif to_docstatus == DocStatus.cancelled(): raise frappe.ValidationError(_("Cannot edit cancelled document")) def set_parent_in_children(self): @@ -885,14 +890,14 @@ class Document(BaseDocument): if (frappe.flags.in_import and frappe.flags.mute_emails) or frappe.flags.in_patch or frappe.flags.in_install: return - if self.flags.notifications_executed==None: + if self.flags.notifications_executed is None: self.flags.notifications_executed = [] from frappe.email.doctype.notification.notification import evaluate_alert - if self.flags.notifications == None: + if self.flags.notifications is None: alerts = frappe.cache().hget('notifications', self.doctype) - if alerts==None: + if alerts is None: alerts = frappe.get_all('Notification', fields=['name', 'event', 'method'], filters={'enabled': 1, 'document_type': self.doctype}) frappe.cache().hset('notifications', self.doctype, alerts) @@ -927,14 +932,14 @@ class Document(BaseDocument): @whitelist.__func__ def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" - self.docstatus = 1 + self.docstatus = DocStatus.submitted() return self.save() @whitelist.__func__ def _cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves. """ - self.docstatus = 2 + self.docstatus = DocStatus.cancelled() return self.save() @whitelist.__func__ @@ -952,7 +957,7 @@ class Document(BaseDocument): frappe.delete_doc(self.doctype, self.name, ignore_permissions = ignore_permissions, flags=self.flags) def run_before_save_methods(self): - """Run standard methods before `INSERT` or `UPDATE`. Standard Methods are: + """Run standard methods before `INSERT` or `UPDATE`. Standard Methods are: - `validate`, `before_save` for **Save**. - `validate`, `before_submit` for **Submit**. @@ -1378,11 +1383,6 @@ class Document(BaseDocument): from frappe.desk.doctype.tag.tag import DocTags return DocTags(self.doctype).get_tags(self.name).split(",")[1:] - def _rename_doc_on_cancel(self): - new_name = gen_new_name_for_cancelled_doc(self) - frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) - self.name = new_name - def __repr__(self): name = self.name or "unsaved" doctype = self.__class__.__name__ diff --git a/frappe/model/dynamic_links.py b/frappe/model/dynamic_links.py index 7311b39b30..03f616ef60 100644 --- a/frappe/model/dynamic_links.py +++ b/frappe/model/dynamic_links.py @@ -32,7 +32,7 @@ def get_dynamic_link_map(for_delete=False): Note: Will not map single doctypes ''' - if getattr(frappe.local, 'dynamic_link_map', None)==None or frappe.flags.in_test: + if getattr(frappe.local, 'dynamic_link_map', None) is None or frappe.flags.in_test: # Build from scratch dynamic_link_map = {} for df in get_dynamic_links(): diff --git a/frappe/model/naming.py b/frappe/model/naming.py index f3d68f3715..b2d11a4cfc 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,14 +1,3 @@ -"""utilities to generate a document name based on various rules defined. - -NOTE: -Till version 13, whenever a submittable document is amended it's name is set to orig_name-X, -where X is a counter and it increments when amended again and so on. - -From Version 14, The naming pattern is changed in a way that amended documents will -have the original name `orig_name` instead of `orig_name-X`. To make this happen -the cancelled document naming pattern is changed to 'orig_name-CANC-X'. -""" - # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE @@ -40,7 +29,7 @@ def set_new_name(doc): doc.name = None if getattr(doc, "amended_from", None): - doc.name = _get_amended_name(doc) + _set_amended_name(doc) return elif getattr(doc.meta, "issingle", False): @@ -256,18 +245,6 @@ def revert_series_if_last(key, name, doc=None): * prefix = #### and hashes = 2021 (hash doesn't exist) * will search hash in key then accordingly get prefix = "" """ - if hasattr(doc, 'amended_from'): - # Do not revert the series if the document is amended. - if doc.amended_from: - return - - # Get document name by parsing incase of fist cancelled document - if doc.docstatus == 2 and not doc.amended_from: - if doc.name.endswith('-CANC'): - name, _ = NameParser.parse_docname(doc.name, sep='-CANC') - else: - name, _ = NameParser.parse_docname(doc.name, sep='-CANC-') - if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: @@ -356,9 +333,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" return value -def _get_amended_name(doc): - name, _ = NameParser(doc).parse_amended_from() - return name +def _set_amended_name(doc): + am_id = 1 + am_prefix = doc.amended_from + if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): + am_id = cint(doc.amended_from.split("-")[-1]) + 1 + am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen + + doc.name = am_prefix + "-" + str(am_id) + return doc.name + def _field_autoname(autoname, doc, skip_slicing=None): """ @@ -399,83 +383,3 @@ def _format_autoname(autoname, doc): name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) return name - -class NameParser: - """Parse document name and return parts of it. - - NOTE: It handles cancellend and amended doc parsing for now. It can be expanded. - """ - def __init__(self, doc): - self.doc = doc - - def parse_amended_from(self): - """ - Cancelled document naming will be in one of these formats - - * original_name-X-CANC - This is introduced to migrate old style naming to new style - * original_name-CANC - This is introduced to migrate old style naming to new style - * original_name-CANC-X - This is the new style naming - - New style naming: In new style naming amended documents will have original name. That says, - when a document gets cancelled we need rename the document by adding `-CANC-X` to the end - so that amended documents can use the original name. - - Old style naming: cancelled documents stay with original name and when amended, amended one - gets a new name as `original_name-X`. To bring new style naming we had to change the existing - cancelled document names and that is done by adding `-CANC` to cancelled documents through patch. - """ - if not getattr(self.doc, 'amended_from', None): - return (None, None) - - # Handle old style cancelled documents (original_name-X-CANC, original_name-CANC) - if self.doc.amended_from.endswith('-CANC'): - name, _ = self.parse_docname(self.doc.amended_from, '-CANC') - amended_from_doc = frappe.get_all( - self.doc.doctype, - filters = {'name': self.doc.amended_from}, - fields = ['amended_from'], - limit=1) - - # Handle format original_name-X-CANC. - if amended_from_doc and amended_from_doc[0].amended_from: - return self.parse_docname(name, '-') - return name, None - - # Handle new style cancelled documents - return self.parse_docname(self.doc.amended_from, '-CANC-') - - @classmethod - def parse_docname(cls, name, sep='-'): - split_list = name.rsplit(sep, 1) - - if len(split_list) == 1: - return (name, None) - return (split_list[0], split_list[1]) - -def get_cancelled_doc_latest_counter(tname, docname): - """Get the latest counter used for cancelled docs of given docname. - """ - name_prefix = f'{docname}-CANC-' - - rows = frappe.db.sql(""" - select - name - from `tab{tname}` - where - name like %(name_prefix)s and docstatus=2 - """.format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1) - - if not rows: - return -1 - return max([int(row.name.replace(name_prefix, '') or -1) for row in rows]) - -def gen_new_name_for_cancelled_doc(doc): - """Generate a new name for cancelled document. - """ - if getattr(doc, "amended_from", None): - name, _ = NameParser(doc).parse_amended_from() - else: - name = doc.name - - counter = get_cancelled_doc_latest_counter(doc.doctype, name) - return f'{name}-CANC-{counter+1}' diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 2cc5818414..6ffaadc5eb 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -80,6 +80,7 @@ def rename_doc( if doctype=='DocType': rename_doctype(doctype, old, new, force) + update_customizations(old, new) update_attachments(doctype, old, new) @@ -174,6 +175,8 @@ def update_user_settings(old, new, link_fields): else: continue +def update_customizations(old: str, new: str) -> None: + frappe.db.set_value("Custom DocPerm", {"parent": old}, "parent", new, update_modified=False) def update_attachments(doctype, old, new): try: diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index e74d88c0f2..1b26cc2c3a 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -1,10 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json import frappe -from frappe.utils import cint from frappe import _ -import json +from frappe.utils import cint +from frappe.model.docstatus import DocStatus class WorkflowStateError(frappe.ValidationError): pass class WorkflowTransitionError(frappe.ValidationError): pass @@ -102,13 +103,13 @@ def apply_workflow(doc, action): doc.set(next_state.update_field, next_state.update_value) new_docstatus = cint(next_state.doc_status) - if doc.docstatus == 0 and new_docstatus == 0: + if doc.docstatus.is_draft() and new_docstatus == DocStatus.draft(): doc.save() - elif doc.docstatus == 0 and new_docstatus == 1: + elif doc.docstatus.is_draft() and new_docstatus == DocStatus.submitted(): doc.submit() - elif doc.docstatus == 1 and new_docstatus == 1: + elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.submitted(): doc.save() - elif doc.docstatus == 1 and new_docstatus == 2: + elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.cancelled(): doc.cancel() else: frappe.throw(_('Illegal Document Status for {0}').format(next_state.state)) @@ -212,10 +213,10 @@ def bulk_workflow_approval(docnames, doctype, action): frappe.db.commit() except Exception as e: if not frappe.message_log: - # Exception is raised manually and not from msgprint or throw + # Exception is raised manually and not from msgprint or throw message = "{0}".format(e.__class__.__name__) if e.args: - message += " : {0}".format(e.args[0]) + message += " : {0}".format(e.args[0]) message_dict = {"docname": docname, "message": message} failed_transactions[docname].append(message_dict) diff --git a/frappe/modules/patch_handler.py b/frappe/modules/patch_handler.py index 8dfb27c0b8..7b635ac940 100644 --- a/frappe/modules/patch_handler.py +++ b/frappe/modules/patch_handler.py @@ -1,37 +1,76 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +""" Patch Handler. + +This file manages execution of manaully written patches. Patches are script +that apply changes in database schema or data to accomodate for changes in the +code. + +Ways to specify patches: + +1. patches.txt file specifies patches that run before doctype schema +migration. Each line represents one patch (old format). +2. patches.txt can alternatively also separate pre and post model sync +patches by using INI like file format: + ```patches.txt + [pre_model_sync] + app.module.patch1 + app.module.patch2 + + + [post_model_sync] + app.module.patch3 + ``` + + When different sections are specified patches are executed in this order: + 1. Run pre_model_sync patches + 2. Reload/resync all doctype schema + 3. Run post_model_sync patches + + Hence any patch that just needs to modify data but doesn't depend on + old schema should be added to post_model_sync section of file. + +3. simple python commands can be added by starting line with `execute:` +`execute:` example: `execute:print("hello world")` """ - Execute Patch Files - To run directly +import configparser +import time +from enum import Enum +from typing import List, Optional - python lib/wnf.py patch patch1, patch2 etc - python lib/wnf.py patch -f patch1, patch2 etc +import frappe - where patch1, patch2 is module name -""" -import frappe, frappe.permissions, time -class PatchError(Exception): pass +class PatchError(Exception): + pass -def run_all(skip_failing=False): + +class PatchType(Enum): + pre_model_sync = "pre_model_sync" + post_model_sync = "post_model_sync" + + +def run_all(skip_failing: bool = False, patch_type: Optional[PatchType] = None) -> None: """run all pending patches""" - executed = [p[0] for p in frappe.db.sql("""select patch from `tabPatch Log`""")] + executed = set(frappe.get_all("Patch Log", fields="patch", pluck="patch")) frappe.flags.final_patches = [] def run_patch(patch): try: if not run_single(patchmodule = patch): - log(patch + ': failed: STOPPED') + print(patch + ': failed: STOPPED') raise PatchError(patch) except Exception: if not skip_failing: raise else: - log('Failed to execute patch') + print('Failed to execute patch') - for patch in get_all_patches(): + patches = get_all_patches(patch_type=patch_type) + + for patch in patches: if patch and (patch not in executed): run_patch(patch) @@ -40,18 +79,57 @@ def run_all(skip_failing=False): patch = patch.replace('finally:', '') run_patch(patch) -def get_all_patches(): +def get_all_patches(patch_type: Optional[PatchType] = None) -> List[str]: + + if patch_type and not isinstance(patch_type, PatchType): + frappe.throw(f"Unsupported patch type specified: {patch_type}") + patches = [] for app in frappe.get_installed_apps(): - if app == "shopping_cart": - continue - # 3-to-4 fix - if app=="webnotes": - app="frappe" - patches.extend(frappe.get_file_items(frappe.get_pymodule_path(app, "patches.txt"))) + patches.extend(get_patches_from_app(app, patch_type=patch_type)) return patches +def get_patches_from_app(app: str, patch_type: Optional[PatchType] = None) -> List[str]: + """ Get patches from an app's patches.txt + + patches.txt can be: + 1. ini like file with section for different patch_type + 2. plain text file with each line representing a patch. + """ + + patches_txt = frappe.get_pymodule_path(app, "patches.txt") + + try: + # Attempt to parse as ini file with pre/post patches + # allow_no_value: patches are not key value pairs + # delimiters = '\n' to avoid treating default `:` and `=` in execute as k:v delimiter + parser = configparser.ConfigParser(allow_no_value=True, delimiters="\n") + # preserve case + parser.optionxform = str + parser.read(patches_txt) + + # empty file + if not parser.sections(): + return [] + + if not patch_type: + return [patch for patch in parser[PatchType.pre_model_sync.value]] + \ + [patch for patch in parser[PatchType.post_model_sync.value]] + + if patch_type.value in parser.sections(): + return [patch for patch in parser[patch_type.value]] + else: + frappe.throw(frappe._("Patch type {} not found in patches.txt").format(patch_type)) + + except configparser.MissingSectionHeaderError: + # treat as old format with each line representing a single patch + # backward compatbility with old patches.txt format + if not patch_type or patch_type == PatchType.pre_model_sync: + return frappe.get_file_items(patches_txt) + + return [] + def reload_doc(args): import frappe.modules run_single(method = frappe.modules.reload_doc, methodargs = args) @@ -73,7 +151,7 @@ def execute_patch(patchmodule, method=None, methodargs=None): frappe.db.begin() start_time = time.time() try: - log('Executing {patch} in {site} ({db})'.format(patch=patchmodule or str(methodargs), + print('Executing {patch} in {site} ({db})'.format(patch=patchmodule or str(methodargs), site=frappe.local.site, db=frappe.db.cur_db_name)) if patchmodule: if patchmodule.startswith("finally:"): @@ -96,7 +174,7 @@ def execute_patch(patchmodule, method=None, methodargs=None): frappe.db.commit() end_time = time.time() block_user(False) - log('Success: Done in {time}s'.format(time = round(end_time - start_time, 3))) + print('Success: Done in {time}s'.format(time = round(end_time - start_time, 3))) return True @@ -109,10 +187,7 @@ def executed(patchmodule): if patchmodule.startswith('finally:'): # patches are saved without the finally: tag patchmodule = patchmodule.replace('finally:', '') - done = frappe.db.get_value("Patch Log", {"patch": patchmodule}) - # if done: - # print "Patch %s already executed in %s" % (patchmodule, frappe.db.cur_db_name) - return done + return frappe.db.get_value("Patch Log", {"patch": patchmodule}) def block_user(block, msg=None): """stop/start execution till patch is run""" @@ -128,6 +203,3 @@ def check_session_stopped(): if frappe.db.get_global("__session_status")=='stop': frappe.msgprint(frappe.db.get_global("__session_status_message")) raise frappe.SessionStopped('Session Stopped') - -def log(msg): - print (msg) diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index bbfd63a277..13b52d2020 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -257,6 +257,12 @@ def make_boilerplate(template, doc, opts=None): pass def get_list(self, args): + pass + + def get_count(self, args): + pass + + def get_stats(self, args): pass""" with open(target_file_path, 'w') as target: diff --git a/frappe/patches.txt b/frappe/patches.txt index af7e4d6e3f..7c2c6d5dc5 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -1,3 +1,4 @@ +[pre_model_sync] frappe.patches.v12_0.remove_deprecated_fields_from_doctype #3 execute:frappe.utils.global_search.setup_global_search_table() execute:frappe.reload_doc('core', 'doctype', 'doctype_action', force=True) #2019-09-23 @@ -87,7 +88,6 @@ frappe.patches.v11_0.set_missing_creation_and_modified_value_for_user_permission frappe.patches.v11_0.set_default_letter_head_source frappe.patches.v12_0.set_primary_key_in_series execute:frappe.delete_doc("Page", "modules", ignore_missing=True) -frappe.patches.v11_0.set_default_letter_head_source frappe.patches.v12_0.setup_comments_from_communications frappe.patches.v12_0.replace_null_values_in_tables frappe.patches.v12_0.reset_home_settings @@ -123,6 +123,9 @@ frappe.patches.v12_0.remove_parent_and_parenttype_from_print_formats frappe.patches.v12_0.remove_example_email_thread_notify execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders() frappe.patches.v12_0.set_correct_url_in_files +execute:frappe.reload_doc('core', 'doctype', 'doctype') +execute:frappe.reload_doc('custom', 'doctype', 'property_setter') +frappe.patches.v13_0.remove_invalid_options_for_data_fields frappe.patches.v13_0.website_theme_custom_scss frappe.patches.v13_0.make_user_type frappe.patches.v13_0.set_existing_dashboard_charts_as_public @@ -153,7 +156,6 @@ frappe.patches.v13_0.rename_notification_fields frappe.patches.v13_0.remove_duplicate_navbar_items frappe.patches.v13_0.set_social_icons frappe.patches.v12_0.set_default_password_reset_limit -execute:frappe.reload_doc('core', 'doctype', 'doctype', force=True) frappe.patches.v13_0.set_route_for_blog_category frappe.patches.v13_0.enable_custom_script frappe.patches.v13_0.update_newsletter_content_type @@ -173,22 +175,24 @@ execute:frappe.delete_doc_if_exists('Page', 'workspace') execute:frappe.delete_doc_if_exists('Page', 'dashboard', force=1) frappe.core.doctype.page.patches.drop_unused_pages execute:frappe.get_doc('Role', 'Guest').save() # remove desk access +frappe.patches.v13_0.remove_chat frappe.patches.v13_0.rename_desk_page_to_workspace # 02.02.2021 frappe.patches.v13_0.delete_package_publish_tool frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings frappe.patches.v13_0.remove_twilio_settings frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns -execute:frappe.reload_doc('core', 'doctype', 'doctype') frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty frappe.patches.v13_0.set_first_day_of_the_week -frappe.patches.v14_0.drop_data_import_legacy -frappe.patches.v14_0.rename_cancelled_documents -frappe.patches.v14_0.copy_mail_data #08.03.21 frappe.patches.v14_0.update_workspace2 # 20.09.2021 +frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021 +frappe.patches.v14_0.transform_todo_schema +frappe.patches.v14_0.remove_post_and_post_comment + +[post_model_sync] +frappe.patches.v14_0.drop_data_import_legacy +frappe.patches.v14_0.copy_mail_data #08.03.21 frappe.patches.v14_0.update_github_endpoints #08-11-2021 frappe.patches.v14_0.remove_db_aggregation -frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021 frappe.patches.v14_0.update_color_names_in_kanban_board_column -frappe.patches.v14_0.transform_todo_schema diff --git a/frappe/patches/v11_0/remove_skip_for_doctype.py b/frappe/patches/v11_0/remove_skip_for_doctype.py index 1bbe74bb6d..6e66c75f68 100644 --- a/frappe/patches/v11_0/remove_skip_for_doctype.py +++ b/frappe/patches/v11_0/remove_skip_for_doctype.py @@ -33,7 +33,7 @@ def execute(): continue skip_for_doctype = user_permission.skip_for_doctype.split('\n') else: # while migrating from v10 -> v11 - if skip_for_doctype_map.get((user_permission.allow, user_permission.user)) == None: + if skip_for_doctype_map.get((user_permission.allow, user_permission.user)) is None: skip_for_doctype = get_doctypes_to_skip(user_permission.allow, user_permission.user) # cache skip for doctype for same user and doctype skip_for_doctype_map[(user_permission.allow, user_permission.user)] = skip_for_doctype diff --git a/frappe/patches/v13_0/remove_chat.py b/frappe/patches/v13_0/remove_chat.py new file mode 100644 index 0000000000..1804c7693f --- /dev/null +++ b/frappe/patches/v13_0/remove_chat.py @@ -0,0 +1,17 @@ +import frappe +import click + +def execute(): + frappe.delete_doc_if_exists("DocType", "Chat Message") + frappe.delete_doc_if_exists("DocType", "Chat Message Attachment") + frappe.delete_doc_if_exists("DocType", "Chat Profile") + frappe.delete_doc_if_exists("DocType", "Chat Token") + frappe.delete_doc_if_exists("DocType", "Chat Room User") + frappe.delete_doc_if_exists("DocType", "Chat Room") + frappe.delete_doc_if_exists("Module Def", "Chat") + + click.secho( + "Chat Module is moved to a separate app and is removed from Frappe in version-13.\n" + "Please install the app to continue using the chat feature: https://github.com/frappe/chat", + fg="yellow", + ) \ No newline at end of file diff --git a/frappe/patches/v13_0/remove_invalid_options_for_data_fields.py b/frappe/patches/v13_0/remove_invalid_options_for_data_fields.py new file mode 100644 index 0000000000..90e4b3c5c6 --- /dev/null +++ b/frappe/patches/v13_0/remove_invalid_options_for_data_fields.py @@ -0,0 +1,17 @@ +# Copyright (c) 2022, Frappe and Contributors +# License: MIT. See LICENSE + + +import frappe +from frappe.model import data_field_options + + +def execute(): + custom_field = frappe.qb.DocType('Custom Field') + (frappe.qb + .update(custom_field) + .set(custom_field.options, None) + .where( + (custom_field.fieldtype == "Data") + & (custom_field.options.notin(data_field_options))) + ).run() diff --git a/frappe/patches/v14_0/copy_mail_data.py b/frappe/patches/v14_0/copy_mail_data.py index d3a5c59209..8ef9cfaf1f 100644 --- a/frappe/patches/v14_0/copy_mail_data.py +++ b/frappe/patches/v14_0/copy_mail_data.py @@ -3,9 +3,6 @@ import frappe def execute(): - frappe.reload_doc("email", "doctype", "imap_folder") - frappe.reload_doc("email", "doctype", "email_account") - # patch for all Email Account with the flag use_imap for email_account in frappe.get_list("Email Account", filters={"enable_incoming": 1, "use_imap": 1}): # get all data from Email Account diff --git a/frappe/patches/v14_0/remove_post_and_post_comment.py b/frappe/patches/v14_0/remove_post_and_post_comment.py new file mode 100644 index 0000000000..3a93139961 --- /dev/null +++ b/frappe/patches/v14_0/remove_post_and_post_comment.py @@ -0,0 +1,5 @@ +import frappe + +def execute(): + frappe.delete_doc_if_exists("DocType", "Post") + frappe.delete_doc_if_exists("DocType", "Post Comment") diff --git a/frappe/patches/v14_0/rename_cancelled_documents.py b/frappe/patches/v14_0/rename_cancelled_documents.py deleted file mode 100644 index 4b565d4f76..0000000000 --- a/frappe/patches/v14_0/rename_cancelled_documents.py +++ /dev/null @@ -1,213 +0,0 @@ -import functools -import traceback - -import frappe - -def execute(): - """Rename cancelled documents by adding a postfix. - """ - rename_cancelled_docs() - -def get_submittable_doctypes(): - """Returns list of submittable doctypes in the system. - """ - return frappe.db.get_all('DocType', filters={'is_submittable': 1}, pluck='name') - -def get_cancelled_doc_names(doctype): - """Return names of cancelled document names those are in old format. - """ - docs = frappe.db.get_all(doctype, filters={'docstatus': 2}, pluck='name') - return [each for each in docs if not (each.endswith('-CANC') or ('-CANC-' in each))] - -@functools.lru_cache() -def get_linked_doctypes(): - """Returns list of doctypes those are linked with given doctype using 'Link' fieldtype. - """ - filters=[['fieldtype','=', 'Link']] - links = frappe.get_all("DocField", - fields=["parent", "fieldname", "options as linked_to"], - filters=filters, - as_list=1) - - links+= frappe.get_all("Custom Field", - fields=["dt as parent", "fieldname", "options as linked_to"], - filters=filters, - as_list=1) - - links_by_doctype = {} - for doctype, fieldname, linked_to in links: - links_by_doctype.setdefault(linked_to, []).append((doctype, fieldname)) - return links_by_doctype - -@functools.lru_cache() -def get_single_doctypes(): - return frappe.get_all("DocType", filters={'issingle': 1}, pluck='name') - -@functools.lru_cache() -def get_dynamic_linked_doctypes(): - filters=[['fieldtype','=', 'Dynamic Link']] - - # find dynamic links of parents - links = frappe.get_all("DocField", - fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], - filters=filters, - as_list=1) - links+= frappe.get_all("Custom Field", - fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], - filters=filters, - as_list=1) - return links - -@functools.lru_cache() -def get_child_tables(): - """ - """ - filters =[['fieldtype', 'in', ('Table', 'Table MultiSelect')]] - links = frappe.get_all("DocField", - fields=["parent as doctype", "options as child_table"], - filters=filters, - as_list=1) - - links+= frappe.get_all("Custom Field", - fields=["dt as doctype", "options as child_table"], - filters=filters, - as_list=1) - - map = {} - for doctype, child_table in links: - map.setdefault(doctype, []).append(child_table) - return map - -def update_cancelled_document_names(doctype, cancelled_doc_names): - return frappe.db.sql(""" - update - `tab{doctype}` - set - name=CONCAT(name, '-CANC') - where - docstatus=2 - and - name in %(cancelled_doc_names)s; - """.format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names}) - -def update_amended_field(doctype, cancelled_doc_names): - return frappe.db.sql(""" - update - `tab{doctype}` - set - amended_from=CONCAT(amended_from, '-CANC') - where - amended_from in %(cancelled_doc_names)s; - """.format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names}) - -def update_attachments(doctype, cancelled_doc_names): - frappe.db.sql(""" - update - `tabFile` - set - attached_to_name=CONCAT(attached_to_name, '-CANC') - where - attached_to_doctype=%(dt)s and attached_to_name in %(cancelled_doc_names)s - """, {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - -def update_versions(doctype, cancelled_doc_names): - frappe.db.sql(""" - UPDATE - `tabVersion` - SET - docname=CONCAT(docname, '-CANC') - WHERE - ref_doctype=%(dt)s AND docname in %(cancelled_doc_names)s - """, {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - -def update_linked_doctypes(doctype, cancelled_doc_names): - single_doctypes = get_single_doctypes() - - for linked_dt, field in get_linked_doctypes().get(doctype, []): - if linked_dt not in single_doctypes: - frappe.db.sql(""" - update - `tab{linked_dt}` - set - `{column}`=CONCAT(`{column}`, '-CANC') - where - `{column}` in %(cancelled_doc_names)s; - """.format(linked_dt=linked_dt, column=field), - {'cancelled_doc_names': cancelled_doc_names}) - else: - doc = frappe.get_single(linked_dt) - if getattr(doc, field) in cancelled_doc_names: - setattr(doc, field, getattr(doc, field)+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def update_dynamic_linked_doctypes(doctype, cancelled_doc_names): - single_doctypes = get_single_doctypes() - - for linked_dt, fieldname, doctype_fieldname in get_dynamic_linked_doctypes(): - if linked_dt not in single_doctypes: - frappe.db.sql(""" - update - `tab{linked_dt}` - set - `{column}`=CONCAT(`{column}`, '-CANC') - where - `{column}` in %(cancelled_doc_names)s and {doctype_fieldname}=%(dt)s; - """.format(linked_dt=linked_dt, column=fieldname, doctype_fieldname=doctype_fieldname), - {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - else: - doc = frappe.get_single(linked_dt) - if getattr(doc, doctype_fieldname) == doctype and getattr(doc, fieldname) in cancelled_doc_names: - setattr(doc, fieldname, getattr(doc, fieldname)+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def update_child_tables(doctype, cancelled_doc_names): - child_tables = get_child_tables().get(doctype, []) - single_doctypes = get_single_doctypes() - - for table in child_tables: - if table not in single_doctypes: - frappe.db.sql(""" - update - `tab{table}` - set - parent=CONCAT(parent, '-CANC') - where - parenttype=%(dt)s and parent in %(cancelled_doc_names)s; - """.format(table=table), {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype}) - else: - doc = frappe.get_single(table) - if getattr(doc, 'parenttype')==doctype and getattr(doc, 'parent') in cancelled_doc_names: - setattr(doc, 'parent', getattr(doc, 'parent')+'-CANC') - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True - doc.save(ignore_permissions=True) - -def rename_cancelled_docs(): - submittable_doctypes = get_submittable_doctypes() - - for dt in submittable_doctypes: - for retry in range(2): - try: - cancelled_doc_names = tuple(get_cancelled_doc_names(dt)) - if not cancelled_doc_names: - break - update_cancelled_document_names(dt, cancelled_doc_names) - update_amended_field(dt, cancelled_doc_names) - update_child_tables(dt, cancelled_doc_names) - update_linked_doctypes(dt, cancelled_doc_names) - update_dynamic_linked_doctypes(dt, cancelled_doc_names) - update_attachments(dt, cancelled_doc_names) - update_versions(dt, cancelled_doc_names) - print(f"Renaming cancelled records of {dt} doctype") - frappe.db.commit() - break - except Exception: - if retry == 1: - print(f"Failed to rename the cancelled records of {dt} doctype, moving on!") - traceback.print_exc() - frappe.db.rollback() - diff --git a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py index ea8a10e43a..ff03604754 100644 --- a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py +++ b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py @@ -5,7 +5,6 @@ from __future__ import unicode_literals import frappe def execute(): - frappe.reload_doc("desk", "doctype", "kanban_board_column") indicator_map = { 'blue': 'Blue', 'orange': 'Orange', diff --git a/frappe/patches/v14_0/update_workspace2.py b/frappe/patches/v14_0/update_workspace2.py index 82076c4328..a4b057b989 100644 --- a/frappe/patches/v14_0/update_workspace2.py +++ b/frappe/patches/v14_0/update_workspace2.py @@ -5,10 +5,10 @@ from frappe import _ def execute(): frappe.reload_doc('desk', 'doctype', 'workspace', force=True) - for seq, wspace in enumerate(frappe.get_all('Workspace', order_by='name asc')): - doc = frappe.get_doc('Workspace', wspace.name) + for seq, workspace in enumerate(frappe.get_all('Workspace', order_by='name asc')): + doc = frappe.get_doc('Workspace', workspace.name) content = create_content(doc) - update_wspace(doc, seq, content) + update_workspace(doc, seq, content) frappe.db.commit() def create_content(doc): @@ -49,7 +49,7 @@ def create_content(doc): del doc.links[doc.links.index(l)] return content -def update_wspace(doc, seq, content): +def update_workspace(doc, seq, content): if not doc.title and not doc.content and not doc.is_standard and not doc.public: doc.sequence_id = seq + 1 doc.content = json.dumps(content) diff --git a/frappe/permissions.py b/frappe/permissions.py index 5faaf7dcfb..af17faba01 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -23,7 +23,7 @@ def print_has_permission_check_logs(func): frappe.flags['has_permission_check_logs'] = [] result = func(*args, **kwargs) self_perm_check = True if not kwargs.get('user') else kwargs.get('user') == frappe.session.user - raise_exception = False if kwargs.get('raise_exception') == False else True + raise_exception = False if kwargs.get('raise_exception') is False else True # print only if access denied # and if user is checking his own permission @@ -559,7 +559,9 @@ def filter_allowed_docs_for_doctype(user_permissions, doctype, with_default_doc= return (allowed_doc, default_doc) if with_default_doc else allowed_doc def push_perm_check_log(log): - if frappe.flags.get('has_permission_check_logs') == None: return + if frappe.flags.get('has_permission_check_logs') is None: + return + frappe.flags.get('has_permission_check_logs').append(_(log)) def has_child_table_permission(child_doctype, ptype="read", child_doc=None, diff --git a/frappe/public/icons/timeless/symbol-defs.svg b/frappe/public/icons/timeless/symbol-defs.svg index dcba913836..4a515d03f9 100644 --- a/frappe/public/icons/timeless/symbol-defs.svg +++ b/frappe/public/icons/timeless/symbol-defs.svg @@ -1,128 +1,309 @@ diff --git a/frappe/public/js/desk.bundle.js b/frappe/public/js/desk.bundle.js index cac02c7a68..e056a34be2 100644 --- a/frappe/public/js/desk.bundle.js +++ b/frappe/public/js/desk.bundle.js @@ -96,6 +96,7 @@ import "./frappe/ui/sort_selector.js"; import "./frappe/change_log.html"; import "./frappe/ui/workspace_loading_skeleton.html"; +import "./frappe/ui/workspace_sidebar_loading_skeleton.html"; import "./frappe/desk.js"; import "./frappe/query_string.js"; diff --git a/frappe/public/js/frappe-web.bundle.js b/frappe/public/js/frappe-web.bundle.js index c962457964..b8d4006090 100644 --- a/frappe/public/js/frappe-web.bundle.js +++ b/frappe/public/js/frappe-web.bundle.js @@ -2,6 +2,7 @@ import "./jquery-bootstrap"; import "./frappe/class.js"; import "./frappe/polyfill.js"; import "./lib/md5.min.js"; +import "./lib/moment.js"; import "./frappe/provide.js"; import "./frappe/format.js"; import "./frappe/utils/number_format.js"; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 2264042539..b153718c70 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -331,7 +331,7 @@ frappe.data_import.ImportPreview = class ImportPreview { is_row_imported(row) { let serial_no = row[0].content; return this.import_log.find(log => { - return log.success && log.row_indexes.includes(serial_no); + return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no); }); } }; diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 202cee645a..51ada70948 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -214,19 +214,20 @@ frappe.Application = class Application { email_password_prompt(email_account,user,i) { var me = this; + const email_id = email_account[i]["email_id"]; let d = new frappe.ui.Dialog({ title: __('Password missing in Email Account'), fields: [ { 'fieldname': 'password', 'fieldtype': 'Password', - 'label': __('Please enter the password for: {0}', [email_account[i]["email_id"]]), + 'label': __('Please enter the password for: {0}', [email_id], "Email Account"), 'reqd': 1 }, { "fieldname": "submit", "fieldtype": "Button", - "label": __("Submit") + "label": __("Submit", null, "Submit password for Email Account") } ] }); diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index 167b4955fa..1b30726a7a 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -534,22 +534,21 @@ export default { }); }, show_google_drive_picker() { - let dialog = cur_dialog; - dialog.hide(); + this.close_dialog = true; let google_drive = new GoogleDrivePicker({ - pickerCallback: data => this.google_drive_callback(data, dialog), + pickerCallback: data => this.google_drive_callback(data), ...this.google_drive_settings }); google_drive.loadPicker(); }, - google_drive_callback(data, dialog) { + google_drive_callback(data) { if (data.action == google.picker.Action.PICKED) { this.upload_file({ file_url: data.docs[0].url, file_name: data.docs[0].name }); } else if (data.action == google.picker.Action.CANCEL) { - dialog.show(); + cur_frm.attachments.new_attachment() } }, url_to_file(url, filename, mime_type) { diff --git a/frappe/public/js/frappe/form/controls/date_range.js b/frappe/public/js/frappe/form/controls/date_range.js index 727e9d55c2..170404f575 100644 --- a/frappe/public/js/frappe/form/controls/date_range.js +++ b/frappe/public/js/frappe/form/controls/date_range.js @@ -11,7 +11,8 @@ frappe.ui.form.ControlDateRange = class ControlDateRange extends frappe.ui.form. language: "en", range: true, autoClose: true, - toggleSelected: false + toggleSelected: false, + firstDay: frappe.datetime.get_first_day_of_the_week_index() }; this.datepicker_options.dateFormat = (frappe.boot.sysdefaults.date_format || 'yyyy-mm-dd'); diff --git a/frappe/public/js/frappe/form/controls/dynamic_link.js b/frappe/public/js/frappe/form/controls/dynamic_link.js index 2c5661ca87..ea9ceb35f3 100644 --- a/frappe/public/js/frappe/form/controls/dynamic_link.js +++ b/frappe/public/js/frappe/form/controls/dynamic_link.js @@ -2,7 +2,7 @@ frappe.ui.form.ControlDynamicLink = class ControlDynamicLink extends frappe.ui.f get_options() { let options = ''; if (this.df.get_options) { - options = this.df.get_options(); + options = this.df.get_options(this); } else if (this.docname==null && cur_dialog) { //for dialog box options = cur_dialog.get_value(this.df.options); diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index ed355cf8b4..9f02485a9e 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -374,10 +374,22 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat } set_custom_query(args) { - var set_nulls = function(obj) { - $.each(obj, function(key, value) { - if(value!==undefined) { - obj[key] = value; + const is_valid_value = (value, key) => { + if (value) return true; + // check if empty value is valid + if (this.frm) { + let field = frappe.meta.get_docfield(this.frm.doctype, key); + // empty value link fields is invalid + return !field || !["Link", "Dynamic Link"].includes(field.fieldtype); + } else { + return value !== undefined; + } + } + + const set_nulls = (obj) => { + $.each(obj, (key, value) => { + if (!is_valid_value(value, key)) { + delete obj[key]; } }); return obj; @@ -458,7 +470,6 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat validate_link_and_fetch(df, options, docname, value) { if (!options) return; - let field_value = ""; const fetch_map = this.fetch_map; const columns_to_fetch = Object.values(fetch_map); @@ -467,16 +478,10 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat return value; } - return frappe.xcall("frappe.client.validate_link", { - doctype: options, - docname: value, - fields: columns_to_fetch, - }).then((response) => { - if (!docname || !columns_to_fetch.length) return response.name; - + function update_dependant_fields(response) { + let field_value = ""; for (const [target_field, source_field] of Object.entries(fetch_map)) { if (value) field_value = response[source_field]; - frappe.model.set_value( df.parent, docname, @@ -485,9 +490,23 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat df.fieldtype, ); } + } - return response.name; - }); + // to avoid unnecessary request + if (value) { + return frappe.xcall("frappe.client.validate_link", { + doctype: options, + docname: value, + fields: columns_to_fetch, + }).then((response) => { + if (!docname || !columns_to_fetch.length) return response.name; + update_dependant_fields(response); + return response.name; + }); + } else { + update_dependant_fields({}); + return value; + } } get fetch_map() { diff --git a/frappe/public/js/frappe/form/footer/base_timeline.js b/frappe/public/js/frappe/form/footer/base_timeline.js index beeba16459..ba7a4eb565 100644 --- a/frappe/public/js/frappe/form/footer/base_timeline.js +++ b/frappe/public/js/frappe/form/footer/base_timeline.js @@ -12,8 +12,11 @@ class BaseTimeline { this.wrapper = this.timeline_wrapper; this.timeline_items_wrapper = $(`
`);
this.timeline_actions_wrapper = $(`
-
-
+
+
`);
@@ -37,7 +40,7 @@ class BaseTimeline {
${label}
`);
action_btn.click(action);
- this.timeline_actions_wrapper.append(action_btn);
+ this.timeline_actions_wrapper.find('.action-buttons').append(action_btn);
return action_btn;
}
diff --git a/frappe/public/js/frappe/form/footer/form_timeline.js b/frappe/public/js/frappe/form/footer/form_timeline.js
index f278d1b64b..d440874f36 100644
--- a/frappe/public/js/frappe/form/footer/form_timeline.js
+++ b/frappe/public/js/frappe/form/footer/form_timeline.js
@@ -77,12 +77,14 @@ class FormTimeline extends BaseTimeline {
const message = __("Add to this activity by mailing to {0}", [link.bold()]);
this.document_email_link_wrapper = $(`
-
+
+
+
+
- ${message}
+
`);
- this.timeline_wrapper.append(this.document_email_link_wrapper);
+ this.timeline_actions_wrapper.append(this.document_email_link_wrapper);
this.document_email_link_wrapper
.find('.document-email-link')
diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js
index 5bc25fbfd5..beefba2e18 100644
--- a/frappe/public/js/frappe/form/form.js
+++ b/frappe/public/js/frappe/form/form.js
@@ -860,36 +860,32 @@ frappe.ui.form.Form = class FrappeForm {
}
_cancel(btn, callback, on_error, skip_confirm) {
+ const me = this;
const cancel_doc = () => {
frappe.validated = true;
- this.script_manager.trigger("before_cancel").then(() => {
+ me.script_manager.trigger("before_cancel").then(() => {
if (!frappe.validated) {
- return this.handle_save_fail(btn, on_error);
+ return me.handle_save_fail(btn, on_error);
}
- const original_name = this.docname;
- const after_cancel = (r) => {
+ var after_cancel = function(r) {
if (r.exc) {
- this.handle_save_fail(btn, on_error);
+ me.handle_save_fail(btn, on_error);
} else {
frappe.utils.play_sound("cancel");
+ me.refresh();
callback && callback();
- this.script_manager.trigger("after_cancel");
- frappe.run_serially([
- () => this.rename_notify(this.doctype, original_name, r.docs[0].name),
- () => frappe.router.clear_re_route(this.doctype, original_name),
- () => this.refresh(),
- ]);
+ me.script_manager.trigger("after_cancel");
}
};
- frappe.ui.form.save(this, "cancel", after_cancel, btn);
+ frappe.ui.form.save(me, "cancel", after_cancel, btn);
});
}
if (skip_confirm) {
cancel_doc();
} else {
- frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error));
+ frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error));
}
}
@@ -911,7 +907,7 @@ frappe.ui.form.Form = class FrappeForm {
'docname': this.doc.name
}).then(is_amended => {
if (is_amended) {
- frappe.throw(__('This document is already amended, you cannot amend it again'));
+ frappe.throw(__('This document is already amended, you cannot ammend it again'));
}
this.validate_form_action("Amend");
var me = this;
@@ -943,7 +939,10 @@ frappe.ui.form.Form = class FrappeForm {
// re-enable buttons
resolve();
}
- frappe.throw (__("No permission to '{0}' {1}", [__(action), __(this.doc.doctype)]));
+
+ frappe.throw(
+ __("No permission to '{0}' {1}", [__(action), __(this.doc.doctype)], "{0} = verb, {1} = object")
+ );
}
}
@@ -1146,8 +1145,7 @@ frappe.ui.form.Form = class FrappeForm {
subject: __(this.meta.name) + ': ' + this.docname,
recipients: this.doc.email || this.doc.email_id || this.doc.contact_email,
attach_document_print: true,
- message: message,
- real_name: this.doc.real_name || this.doc.contact_display || this.doc.contact_name
+ message: message
});
}
diff --git a/frappe/public/js/frappe/form/form_viewers.js b/frappe/public/js/frappe/form/form_viewers.js
index 964576ef8a..ecf5eea504 100644
--- a/frappe/public/js/frappe/form/form_viewers.js
+++ b/frappe/public/js/frappe/form/form_viewers.js
@@ -27,19 +27,40 @@ frappe.ui.form.FormViewers.set_users = function(data, type) {
const users = data.users || [];
const new_users = users.filter(user => !past_users.includes(user));
- frappe.model.set_docinfo(doctype, docname, type, {
- past: past_users.concat(new_users),
- new: new_users,
- current: users
- });
+ if (new_users.length===0) return;
- if (
- cur_frm &&
- cur_frm.doc &&
- cur_frm.doc.doctype === doctype &&
- cur_frm.doc.name == docname &&
- cur_frm.viewers
- ) {
- cur_frm.viewers.refresh(true, type);
+ const set_and_refresh = () => {
+ const info = {
+ past: past_users.concat(new_users),
+ new: new_users,
+ current: users
+ };
+
+ frappe.model.set_docinfo(doctype, docname, type, info);
+
+ if (
+ cur_frm &&
+ cur_frm.doc &&
+ cur_frm.doc.doctype === doctype &&
+ cur_frm.doc.name == docname &&
+ cur_frm.viewers
+ ) {
+ cur_frm.viewers.refresh(true, type);
+ }
+ };
+
+ let unknown_users = [];
+ for (let user of users) {
+ if (!frappe.boot.user_info[user]) unknown_users.push(user);
+ }
+
+ if (unknown_users.length===0) {
+ set_and_refresh();
+ } else {
+ // load additional user info
+ frappe.xcall('frappe.desk.form.load.get_user_info_for_viewers', {users: unknown_users}).then((data) => {
+ Object.assign(frappe.boot.user_info, data);
+ set_and_refresh();
+ });
}
};
diff --git a/frappe/public/js/frappe/form/multi_select_dialog.js b/frappe/public/js/frappe/form/multi_select_dialog.js
index bc0286e62d..bc5f7a9b52 100644
--- a/frappe/public/js/frappe/form/multi_select_dialog.js
+++ b/frappe/public/js/frappe/form/multi_select_dialog.js
@@ -150,8 +150,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
});
}
+ is_child_selection_enabled() {
+ return this.dialog.fields_dict['allow_child_item_selection'].get_value();
+ }
+
toggle_child_selection() {
- if (this.dialog.fields_dict['allow_child_item_selection'].get_value()) {
+ if (this.is_child_selection_enabled()) {
this.show_child_results();
} else {
this.child_results = [];
@@ -289,7 +293,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
parent: this.dialog.get_field('filter_area').$wrapper,
doctype: this.doctype,
on_change: () => {
- this.get_results();
+ if (this.is_child_selection_enabled()) {
+ this.show_child_results();
+ } else {
+ this.get_results();
+ }
}
});
// 'Apply Filter' breaks since the filers are not in a popover
@@ -325,7 +333,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
this.$parent.find('.input-with-feedback').on('change', () => {
frappe.flags.auto_scroll = false;
- this.get_results();
+ if (this.is_child_selection_enabled()) {
+ this.show_child_results();
+ } else {
+ this.get_results();
+ }
});
this.$parent.find('[data-fieldtype="Data"]').on('input', () => {
@@ -333,8 +345,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
clearTimeout($this.data('timeout'));
$this.data('timeout', setTimeout(function () {
frappe.flags.auto_scroll = false;
- me.empty_list();
- me.get_results();
+ if (me.is_child_selection_enabled()) {
+ me.show_child_results();
+ } else {
+ me.empty_list();
+ me.get_results();
+ }
}, 300));
});
}
diff --git a/frappe/public/js/frappe/form/save.js b/frappe/public/js/frappe/form/save.js
index 65d84e2202..934c90f017 100644
--- a/frappe/public/js/frappe/form/save.js
+++ b/frappe/public/js/frappe/form/save.js
@@ -7,12 +7,12 @@ frappe.ui.form.save = function (frm, action, callback, btn) {
$(btn).prop("disabled", true);
// specified here because there are keyboard shortcuts to save
- var working_label = {
- "Save": __("Saving"),
- "Submit": __("Submitting"),
- "Update": __("Updating"),
- "Amend": __("Amending"),
- "Cancel": __("Cancelling")
+ const working_label = {
+ "Save": __("Saving", null, "Freeze message while saving a document"),
+ "Submit": __("Submitting", null, "Freeze message while submitting a document"),
+ "Update": __("Updating", null, "Freeze message while updating a document"),
+ "Amend": __("Amending", null, "Freeze message while amending a document"),
+ "Cancel": __("Cancelling", null, "Freeze message while cancelling a document"),
}[toTitle(action)];
var freeze_message = working_label ? __(working_label) : "";
@@ -154,8 +154,8 @@ frappe.ui.form.save = function (frm, action, callback, btn) {
if (error_fields.length) {
let meta = frappe.get_meta(doc.doctype);
if (meta.istable) {
- var message = __('Mandatory fields required in table {0}, Row {1}',
- [__(frappe.meta.docfield_map[doc.parenttype][doc.parentfield].label).bold(), doc.idx]);
+ const table_label = __(frappe.meta.docfield_map[doc.parenttype][doc.parentfield].label).bold();
+ var message = __('Mandatory fields required in table {0}, Row {1}', [table_label, doc.idx]);
} else {
var message = __('Mandatory fields required in {0}', [__(doc.doctype)]);
}
@@ -276,4 +276,3 @@ frappe.ui.form.update_calling_link = (newdoc) => {
frappe._from_link = null;
}
}
-
diff --git a/frappe/public/js/frappe/form/script_manager.js b/frappe/public/js/frappe/form/script_manager.js
index d1732ee702..6169fa75b8 100644
--- a/frappe/public/js/frappe/form/script_manager.js
+++ b/frappe/public/js/frappe/form/script_manager.js
@@ -192,7 +192,7 @@ frappe.ui.form.ScriptManager = class ScriptManager {
}
function setup_add_fetch(df) {
- if ((['Data', 'Read Only', 'Text', 'Small Text', 'Currency', 'Check',
+ if ((['Data', 'Read Only', 'Text', 'Small Text', 'Currency', 'Check', 'Attach Image',
'Text Editor', 'Code', 'Link', 'Float', 'Int', 'Date', 'Select', 'Duration'].includes(df.fieldtype) || df.read_only==1)
&& df.fetch_from && df.fetch_from.indexOf(".")!=-1) {
var parts = df.fetch_from.split(".");
diff --git a/frappe/public/js/frappe/form/templates/form_sidebar.html b/frappe/public/js/frappe/form/templates/form_sidebar.html
index efa8b9ea5e..dcea2f4647 100644
--- a/frappe/public/js/frappe/form/templates/form_sidebar.html
+++ b/frappe/public/js/frappe/form/templates/form_sidebar.html
@@ -1,5 +1,5 @@
-
+ ${message}
+