From 076816a7d7d7541b66cc7c3d6a977256b77642dd Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 10 Jun 2022 15:55:34 +0530 Subject: [PATCH 01/21] fix: Allow to set empty values in Bulk Edit - minor UX fixes --- .../public/js/frappe/list/bulk_operations.js | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/list/bulk_operations.js b/frappe/public/js/frappe/list/bulk_operations.js index 94ec9d4e67..7c8c515643 100644 --- a/frappe/public/js/frappe/list/bulk_operations.js +++ b/frappe/public/js/frappe/list/bulk_operations.js @@ -208,7 +208,7 @@ export default class BulkOperations { const default_field = field_options.find(value => status_regex.test(value)); const dialog = new frappe.ui.Dialog({ - title: __('Edit'), + title: __('Bulk Edit'), fields: [ { 'fieldtype': 'Select', @@ -225,7 +225,9 @@ export default class BulkOperations { 'fieldtype': 'Data', 'label': __('Value'), 'fieldname': 'value', - 'reqd': 1 + onchange() { + show_help_text(); + } } ], primary_action: ({ value }) => { @@ -239,7 +241,7 @@ export default class BulkOperations { docnames: docnames, action: 'update', data: { - [fieldname]: value + [fieldname]: value || null } } }).then(r => { @@ -254,10 +256,11 @@ export default class BulkOperations { frappe.show_alert(__('Updated successfully')); }); }, - primary_action_label: __('Update') + primary_action_label: __('Update {0} records', [docnames.length]), }); if (default_field) set_value_field(dialog); // to set `Value` df based on default `Field` + show_help_text(); function set_value_field (dialogObj) { const new_df = Object.assign({}, @@ -275,9 +278,20 @@ export default class BulkOperations { new_df.default = options[0] || options[1]; } new_df.label = __('Value'); - new_df.reqd = 1; + new_df.onchange = show_help_text; + delete new_df.depends_on; dialogObj.replace_field('value', new_df); + show_help_text(); + } + + function show_help_text() { + let value = dialog.get_value('value'); + if (value == null || value === '') { + dialog.set_df_property('value', 'description', __('You have not entered a value. The field will be set to empty.')); + } else { + dialog.set_df_property('value', 'description', ''); + } } dialog.refresh(); From c2b9197624efbb66a2f0a7743c96d7ed163cd768 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 10 Jun 2022 20:01:33 +0530 Subject: [PATCH 02/21] fix: default tab fieldname conflict --- frappe/public/js/frappe/form/layout.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 403abf0981..c1c9967507 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -122,7 +122,8 @@ frappe.ui.form.Layout = class Layout { } if (this.is_tabbed_layout()) { - let default_tab = {label: __('Details'), fieldname: 'details', fieldtype: "Tab Break"}; + // add a tab without `fieldname` to avoid conflicts + let default_tab = {label: __('Details'), fieldtype: "Tab Break"}; let first_tab = this.fields[1].fieldtype === "Tab Break" ? this.fields[1] : null; if (!first_tab) { this.fields.splice(1, 0, default_tab); From 89889753e4dadab618bdbf8101823e8187cc995d Mon Sep 17 00:00:00 2001 From: Florian HENRY Date: Sun, 12 Jun 2022 12:11:25 +0200 Subject: [PATCH 03/21] fix: JS error in page.js when the is a quote in button translation --- frappe/public/js/frappe/ui/page.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/page.js b/frappe/public/js/frappe/ui/page.js index 70e0862ba5..eded1aefc5 100644 --- a/frappe/public/js/frappe/ui/page.js +++ b/frappe/public/js/frappe/ui/page.js @@ -510,7 +510,7 @@ frappe.ui.Page = class Page { if (!label || !parent) return false; - const item_selector = `${selector}[data-label='${encodeURIComponent(label)}']`; + const item_selector = `${selector}[data-label="${encodeURIComponent(label)}"]`; const existing_items = $(parent).find(item_selector); return existing_items?.length > 0 && existing_items; From 03f48580ef1baa91b3d1bd7fdb90e20fa3f9adb4 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 12 Jun 2022 16:24:39 +0530 Subject: [PATCH 04/21] test: button with single quote in label --- cypress/integration/custom_buttons.js | 1 + 1 file changed, 1 insertion(+) diff --git a/cypress/integration/custom_buttons.js b/cypress/integration/custom_buttons.js index e2f02668e9..6045d009c2 100644 --- a/cypress/integration/custom_buttons.js +++ b/cypress/integration/custom_buttons.js @@ -4,6 +4,7 @@ const test_button_names = [ "Porcupine Tree (the GOAT)", "AC / DC", `Electronic Dance "music"`, + "l'imperatrice", ]; const add_button = (label, group = "TestGroup") => { From 21442f5cbaa5c7b1a5a0aa1c8a376ddbc6d04ad6 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 12 Jun 2022 17:19:13 +0530 Subject: [PATCH 05/21] perf: disable creating version for new docs Each new doc inserts a version, this contains nothing but creator and creation time.. which is already immutable information on the original document. This was added for cases like data import to track from where document got created, ref: https://github.com/frappe/frappe/commit/b7dfe7969dd6b2851844b1e9c090cbb6447748de Fix: only add a version on creation IF creation info is present on flags --- frappe/core/doctype/version/test_version.py | 13 +++++++++++++ frappe/core/doctype/version/version.py | 19 ++++++++++++++++--- frappe/model/document.py | 7 +++---- frappe/tests/test_form_load.py | 2 +- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/version/test_version.py b/frappe/core/doctype/version/test_version.py index c35430b17b..3e82f30f06 100644 --- a/frappe/core/doctype/version/test_version.py +++ b/frappe/core/doctype/version/test_version.py @@ -32,6 +32,19 @@ class TestVersion(unittest.TestCase): self.assertEqual(get_old_values(diff)[1], "01-01-2014 00:00:00") self.assertEqual(get_new_values(diff)[1], "07-20-2017 00:00:00") + def test_no_version_on_new_doc(self): + from frappe.desk.form.load import get_versions + + t = frappe.get_doc(doctype="ToDo", description="something") + t.save(ignore_version=False) + + self.assertFalse(get_versions(t)) + + t = frappe.get_doc(t.doctype, t.name) + t.description = "changed" + t.save(ignore_version=False) + self.assertTrue(get_versions(t)) + def get_fieldnames(change_array): return [d[0] for d in change_array] diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 863885e85c..fa6ba0a9cf 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import json +from typing import Optional import frappe from frappe.model import no_value_fields, table_fields @@ -9,7 +10,15 @@ from frappe.model.document import Document class Version(Document): - def set_diff(self, old, new): + def update_version_info(self, old: Optional[Document], new: Document) -> bool: + """Update changed info and return true if change contains useful data.""" + if not old: + # Check if doc has some information about creation source like data import + return self.for_insert(new) + else: + return self.set_diff(old, new) + + def set_diff(self, old: Document, new: Document) -> bool: """Set the data property with the diff of the docs if present""" diff = get_diff(old, new) if diff: @@ -20,8 +29,11 @@ class Version(Document): else: return False - def for_insert(self, doc): + def for_insert(self, doc: Document) -> bool: updater_reference = doc.flags.updater_reference + if not updater_reference: + return False + data = { "creation": doc.creation, "updater_reference": updater_reference, @@ -29,7 +41,8 @@ class Version(Document): } self.ref_doctype = doc.doctype self.docname = doc.name - self.data = frappe.as_json(data) + self.data = frappe.as_json(data, indent=None, separators=(",", ":")) + return True def get_data(self): return json.loads(self.data) diff --git a/frappe/model/document.py b/frappe/model/document.py index fa1f423d11..a98d49e58c 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1198,11 +1198,10 @@ class Document(BaseDocument): return version = frappe.new_doc("Version") - if not self._doc_before_save: - version.for_insert(self) - version.insert(ignore_permissions=True) - elif version.set_diff(self._doc_before_save, self): + + if is_useful_diff := version.update_version_info(self._doc_before_save, self): version.insert(ignore_permissions=True) + if not frappe.flags.in_migrate: # follow since you made a change? if frappe.get_cached_value("User", frappe.session.user, "follow_created_documents"): diff --git a/frappe/tests/test_form_load.py b/frappe/tests/test_form_load.py index e92b8c3ff2..22db56eeef 100644 --- a/frappe/tests/test_form_load.py +++ b/frappe/tests/test_form_load.py @@ -181,7 +181,7 @@ class TestFormLoad(unittest.TestCase): self.assertEqual(len(docinfo.comments), 1) self.assertIn("test", docinfo.comments[0].content) - self.assertGreaterEqual(len(docinfo.versions), 2) + self.assertGreaterEqual(len(docinfo.versions), 1) self.assertEqual(set(docinfo.tags.split(",")), {"more_tag", "test_tag"}) From a4f2912fdf5e0945d89fe588f5cb6b02cc3c2582 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 13 Jun 2022 11:11:47 +0530 Subject: [PATCH 06/21] fix: Handle case where document title can be NONE (cherry picked from commit 9b67fc24bc290789158f37a8f2ce10b505878792) --- frappe/www/printview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 85ffc671fa..c8595a6f2c 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -63,7 +63,7 @@ def get_context(context): "body": body, "print_style": print_style, "comment": frappe.session.user, - "title": frappe.utils.strip_html(doc.get_title()), + "title": frappe.utils.strip_html(doc.get_title() or doc.name), "lang": frappe.local.lang, "layout_direction": "rtl" if is_rtl() else "ltr", "doctype": frappe.form_dict.doctype, From 53a079f1017e883bb07c745135db9506705b3841 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 13 Jun 2022 11:12:53 +0530 Subject: [PATCH 07/21] fix: `doc.get_title` should return empty string if title is not set (cherry picked from commit b244c9148192362de43aa483b328709745f3be82) --- frappe/model/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index a98d49e58c..22514df75a 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -438,7 +438,7 @@ class Document(BaseDocument): def get_title(self): """Get the document title based on title_field or `title` or `name`""" - return self.get(self.meta.get_title_field()) + return self.get(self.meta.get_title_field()) or "" def set_title_field(self): """Set title field based on template""" From 63a5db94cfa728b6116e2a806ae8f2fc7988e4d8 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Mon, 13 Jun 2022 12:03:39 +0530 Subject: [PATCH 08/21] chore: Add mergify[bot] to exception list https://user-images.githubusercontent.com/13928957/173293263-4987c494-4524-46dd-996c-36f1ca760c68.png According to this mergify's login id is `mergify[bot]` so am guessing this should work. --- .mergify.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.mergify.yml b/.mergify.yml index f1333362a8..97df91a927 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -7,6 +7,7 @@ pull_request_rules: - author!=gavindsouza - author!=deepeshgarg007 - author!=ankush + - author!=mergify[bot] - or: - base=version-13 - base=version-12 From eea5cc52da8d246629fa4c7793745faf35f15276 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 13 Jun 2022 16:24:00 +0530 Subject: [PATCH 09/21] refactor!: Remove Data Migration Tool (#17134) * refactor: Remove Data Migration Tool * chore: unnecessary imports --- frappe/data_migration/__init__.py | 0 frappe/data_migration/doctype/__init__.py | 0 .../data_migration_connector/__init__.py | 0 .../connectors/__init__.py | 0 .../connectors/base.py | 24 - .../connectors/frappe_connection.py | 32 - .../data_migration_connector.js | 47 - .../data_migration_connector.json | 307 ------- .../data_migration_connector.py | 107 --- .../test_data_migration_connector.py | 8 - .../data_migration_mapping/__init__.py | 0 .../data_migration_mapping.js | 8 - .../data_migration_mapping.json | 456 ---------- .../data_migration_mapping.py | 83 -- .../test_data_migration_mapping.py | 8 - .../data_migration_mapping_detail/__init__.py | 0 .../data_migration_mapping_detail.json | 163 ---- .../data_migration_mapping_detail.py | 9 - .../doctype/data_migration_plan/__init__.py | 0 .../data_migration_plan.js | 10 - .../data_migration_plan.json | 224 ----- .../data_migration_plan.py | 78 -- .../test_data_migration_plan.py | 8 - .../data_migration_plan_mapping/__init__.py | 0 .../data_migration_plan_mapping.json | 103 --- .../data_migration_plan_mapping.py | 9 - .../doctype/data_migration_run/__init__.py | 0 .../data_migration_run/data_migration_run.js | 14 - .../data_migration_run.json | 838 ------------------ .../data_migration_run/data_migration_run.py | 514 ----------- .../test_data_migration_run.py | 128 --- frappe/model/sync.py | 18 - frappe/modules.txt | 1 - frappe/patches.txt | 1 + .../v14_0/delete_data_migration_tool.py | 12 + 35 files changed, 13 insertions(+), 3197 deletions(-) delete mode 100644 frappe/data_migration/__init__.py delete mode 100644 frappe/data_migration/doctype/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/connectors/base.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js delete mode 100644 frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json delete mode 100644 frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json delete mode 100644 frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js delete mode 100644 frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json delete mode 100644 frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json delete mode 100644 frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py delete mode 100644 frappe/data_migration/doctype/data_migration_run/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_run/data_migration_run.js delete mode 100644 frappe/data_migration/doctype/data_migration_run/data_migration_run.json delete mode 100644 frappe/data_migration/doctype/data_migration_run/data_migration_run.py delete mode 100644 frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py create mode 100644 frappe/patches/v14_0/delete_data_migration_tool.py diff --git a/frappe/data_migration/__init__.py b/frappe/data_migration/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/__init__.py b/frappe/data_migration/doctype/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/__init__.py b/frappe/data_migration/doctype/data_migration_connector/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py b/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py deleted file mode 100644 index 7d2b320c59..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py +++ /dev/null @@ -1,24 +0,0 @@ -from abc import ABCMeta, abstractmethod - -from frappe.utils.password import get_decrypted_password - - -class BaseConnection(metaclass=ABCMeta): - @abstractmethod - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - @abstractmethod - def insert(self, doctype, doc): - pass - - @abstractmethod - def update(self, doctype, doc, migration_id): - pass - - @abstractmethod - def delete(self, doctype, migration_id): - pass - - def get_password(self): - return get_decrypted_password("Data Migration Connector", self.connector.name) diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py deleted file mode 100644 index 8228529562..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py +++ /dev/null @@ -1,32 +0,0 @@ -import frappe -from frappe.frappeclient import FrappeClient - -from .base import BaseConnection - - -class FrappeConnection(BaseConnection): - def __init__(self, connector): - self.connector = connector - self.connection = FrappeClient( - self.connector.hostname, self.connector.username, self.get_password() - ) - self.name_field = "name" - - def insert(self, doctype, doc): - doc = frappe._dict(doc) - doc.doctype = doctype - return self.connection.insert(doc) - - def update(self, doctype, doc, migration_id): - doc = frappe._dict(doc) - doc.doctype = doctype - doc.name = migration_id - return self.connection.update(doc) - - def delete(self, doctype, migration_id): - return self.connection.delete(doctype, migration_id) - - def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20): - return self.connection.get_list( - doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length - ) diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js deleted file mode 100644 index 0898fcf4e7..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Connector', { - onload(frm) { - if(frappe.boot.developer_mode) { - frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm)); - } - }, - new_connection(frm) { - const d = new frappe.ui.Dialog({ - title: __('New Connection'), - fields: [ - { label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 }, - { label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 }, - ], - primary_action_label: __('Create'), - primary_action: (values) => { - let { module, connection_name } = values; - - frm.events.create_new_connection(module, connection_name) - .then(r => { - if (r.message) { - const connector_name = connection_name - .replace('connection', 'Connector') - .replace('Connection', 'Connector') - .trim(); - - frm.set_value('connector_name', connector_name); - frm.set_value('connector_type', 'Custom'); - frm.set_value('python_module', r.message); - frm.save(); - frappe.show_alert(__("New module created {0}", [r.message])); - d.hide(); - } - }); - } - }); - - d.show(); - }, - create_new_connection(module, connection_name) { - return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', { - module, connection_name - }); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json deleted file mode 100644 index 338d59aadd..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:connector_name", - "beta": 1, - "creation": "2017-08-11 05:03:27.091416", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "connector_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:!doc.is_custom", - "fieldname": "connector_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Type", - "length": 0, - "no_copy": 0, - "options": "\nFrappe\nCustom", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.connector_type == 'Custom'", - "fieldname": "python_module", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Python Module", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "authentication_credentials", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Authentication Credentials", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "", - "fieldname": "hostname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Hostname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "database_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Database Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "username", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Username", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "password", - "fieldtype": "Password", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Password", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-12-01 13:38:55.992499", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Connector", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py deleted file mode 100644 index 9db7fc2445..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import os - -import frappe -from frappe import _ -from frappe.model.document import Document -from frappe.modules.export_file import create_init_py - -from .connectors.base import BaseConnection -from .connectors.frappe_connection import FrappeConnection - - -class DataMigrationConnector(Document): - def validate(self): - if not (self.python_module or self.connector_type): - frappe.throw(_("Enter python module or select connector type")) - - if self.python_module: - try: - get_connection_class(self.python_module) - except: - frappe.throw(frappe._("Invalid module path")) - - def get_connection(self): - if self.python_module: - _class = get_connection_class(self.python_module) - return _class(self) - else: - self.connection = FrappeConnection(self) - - return self.connection - - -@frappe.whitelist() -def create_new_connection(module, connection_name): - if not frappe.conf.get("developer_mode"): - frappe.msgprint(_("Please enable developer mode to create new connection")) - return - # create folder - module_path = frappe.get_module_path(module) - connectors_folder = os.path.join(module_path, "connectors") - frappe.create_folder(connectors_folder) - - # create init py - create_init_py(module_path, "connectors", "") - - connection_class = connection_name.replace(" ", "") - file_name = frappe.scrub(connection_name) + ".py" - file_path = os.path.join(module_path, "connectors", file_name) - - # create boilerplate file - with open(file_path, "w") as f: - f.write(connection_boilerplate.format(connection_class=connection_class)) - - # get python module string from file_path - app_name = frappe.db.get_value("Module Def", module, "app_name") - python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace( - os.path.sep, "." - )[:-3] - - return python_module - - -def get_connection_class(python_module): - filename = python_module.rsplit(".", 1)[-1] - classname = frappe.unscrub(filename).replace(" ", "") - module = frappe.get_module(python_module) - - raise_error = False - if hasattr(module, classname): - _class = getattr(module, classname) - if not issubclass(_class, BaseConnection): - raise_error = True - else: - raise_error = True - - if raise_error: - raise ImportError(filename) - - return _class - - -connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection - -class {connection_class}(BaseConnection): - def __init__(self, connector): - # self.connector = connector - # self.connection = YourModule(self.connector.username, self.get_password()) - # self.name_field = 'id' - pass - - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - def insert(self, doctype, doc): - pass - - def update(self, doctype, doc, migration_id): - pass - - def delete(self, doctype, migration_id): - pass - -""" diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py deleted file mode 100644 index c4090796ab..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationConnector(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js deleted file mode 100644 index 6c99b9a54d..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Mapping', { - refresh: function() { - - } -}); diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json deleted file mode 100644 index 998abdf6ca..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json +++ /dev/null @@ -1,456 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:mapping_name", - "beta": 1, - "creation": "2017-08-11 05:11:49.975801", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_objectname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Objectname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local DocType", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Local Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_5", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull\nSync", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "10", - "fieldname": "page_length", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page Length", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "migration_id_field", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Migration ID Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "fields", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Field Maps", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping Detail", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 1, - "columns": 0, - "fieldname": "condition_detail", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition Detail", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "condition", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-09-27 18:06:43.275207", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py deleted file mode 100644 index 49af65e99b..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.model.document import Document -from frappe.utils.safe_exec import get_safe_globals - - -class DataMigrationMapping(Document): - def get_filters(self): - if self.condition: - return frappe.safe_eval(self.condition, get_safe_globals()) - - def get_fields(self): - fields = [] - for f in self.fields: - if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")): - fields.append(f.local_fieldname) - - if frappe.db.has_column(self.local_doctype, self.migration_id_field): - fields.append(self.migration_id_field) - - if "name" not in fields: - fields.append("name") - - return fields - - def get_mapped_record(self, doc): - """Build a mapped record using information from the fields table""" - mapped = frappe._dict() - - key_fieldname = "remote_fieldname" - value_fieldname = "local_fieldname" - - if self.mapping_type == "Pull": - key_fieldname, value_fieldname = value_fieldname, key_fieldname - - for field_map in self.fields: - key = get_source_value(field_map, key_fieldname) - - if not field_map.is_child_table: - # field to field mapping - value = get_value_from_fieldname(field_map, value_fieldname, doc) - else: - # child table mapping - mapping_name = field_map.child_table_mapping - value = get_mapped_child_records( - mapping_name, doc.get(get_source_value(field_map, value_fieldname)) - ) - - mapped[key] = value - - return mapped - - -def get_mapped_child_records(mapping_name, child_docs): - mapped_child_docs = [] - mapping = frappe.get_doc("Data Migration Mapping", mapping_name) - for child_doc in child_docs: - mapped_child_docs.append(mapping.get_mapped_record(child_doc)) - - return mapped_child_docs - - -def get_value_from_fieldname(field_map, fieldname_field, doc): - field_name = get_source_value(field_map, fieldname_field) - - if field_name.startswith("eval:"): - value = frappe.safe_eval(field_name[5:], get_safe_globals()) - elif field_name[0] in ('"', "'"): - value = field_name[1:-1] - else: - value = get_source_value(doc, field_name) - return value - - -def get_source_value(source, key): - """Get value from source (object or dict) based on key""" - if isinstance(source, dict): - return source.get(key) - else: - return getattr(source, key) diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py deleted file mode 100644 index 30d2a6bcfe..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationMapping(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py b/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json deleted file mode 100644 index ede9213f14..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-08-11 05:09:10.900237", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "is_child_table", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Is Child Table", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "is_child_table", - "fieldname": "child_table_mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Child Table Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-28 17:13:31.337005", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping Detail", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py deleted file mode 100644 index abd6348a26..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationMappingDetail(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan/__init__.py b/frappe/data_migration/doctype/data_migration_plan/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js deleted file mode 100644 index 357ef2972f..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Plan', { - onload(frm) { - frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', { - data_migration_plan: frm.doc.name - })); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json deleted file mode 100644 index 2cfc2e3bd7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:plan_name", - "beta": 0, - "creation": "2017-08-11 05:15:51.482165", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "plan_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Plan Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 1 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Module", - "length": 0, - "no_copy": 0, - "options": "Module Def", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mappings", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mappings", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "preprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Preprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "postprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Postprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py deleted file mode 100644 index 4118e8e7fe..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.model.document import Document -from frappe.modules import get_module_path, scrub_dt_dn -from frappe.modules.export_file import create_init_py, export_to_files - - -def get_mapping_module(module, mapping_name): - app_name = frappe.db.get_value("Module Def", module, "app_name") - mapping_name = frappe.scrub(mapping_name) - module = frappe.scrub(module) - - try: - return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}") - except ImportError: - return None - - -class DataMigrationPlan(Document): - def on_update(self): - # update custom fields in mappings - self.make_custom_fields_for_mappings() - - if frappe.flags.in_import or frappe.flags.in_test: - return - - if frappe.local.conf.get("developer_mode"): - record_list = [["Data Migration Plan", self.name]] - - for m in self.mappings: - record_list.append(["Data Migration Mapping", m.mapping]) - - export_to_files(record_list=record_list, record_module=self.module) - - for m in self.mappings: - dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping) - create_init_py(get_module_path(self.module), dt, dn) - - def make_custom_fields_for_mappings(self): - frappe.flags.ignore_in_install = True - label = self.name + " ID" - fieldname = frappe.scrub(label) - - df = { - "label": label, - "fieldname": fieldname, - "fieldtype": "Data", - "hidden": 1, - "read_only": 1, - "unique": 1, - "no_copy": 1, - } - - for m in self.mappings: - mapping = frappe.get_doc("Data Migration Mapping", m.mapping) - create_custom_field(mapping.local_doctype, df) - mapping.migration_id_field = fieldname - mapping.save() - - # Create custom field in Deleted Document - create_custom_field("Deleted Document", df) - frappe.flags.ignore_in_install = False - - def pre_process_doc(self, mapping_name, doc): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "pre_process"): - return module.pre_process(doc) - return doc - - def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "post_process"): - return module.post_process(local_doc=local_doc, remote_doc=remote_doc) diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py deleted file mode 100644 index ef3bfa3a70..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationPlan(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json deleted file mode 100644 index 5acf014715..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 1, - "creation": "2017-08-11 05:15:38.390831", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "enabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Enabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-20 21:43:04.908650", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py deleted file mode 100644 index 0650f4b2c7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationPlanMapping(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_run/__init__.py b/frappe/data_migration/doctype/data_migration_run/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js b/frappe/data_migration/doctype/data_migration_run/data_migration_run.js deleted file mode 100644 index 82323c62f1..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Run', { - refresh: function(frm) { - if (frm.doc.status !== 'Success') { - frm.add_custom_button(__('Run'), () => frm.call('run')); - } - if (frm.doc.status === 'Started') { - frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete, - __('Currently updating {0}', [frm.doc.current_mapping])); - } - } -}); diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json b/frappe/data_migration/doctype/data_migration_run/data_migration_run.json deleted file mode 100644 index db77997928..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json +++ /dev/null @@ -1,838 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-09-11 12:55:27.597728", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_plan", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Data Migration Plan", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_connector", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data Migration Connector", - "length": 0, - "no_copy": 0, - "options": "Data Migration Connector", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 1, - "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "start_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Start Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "end_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "End Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Remote ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping", - "length": 0, - "no_copy": 1, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_delete_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Delete Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "current_mapping_action", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Action", - "length": 0, - "no_copy": 1, - "options": "Insert\nDelete", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "total_pages", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Total Pages", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "percent_complete", - "fieldtype": "Percent", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Percent Complete", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "trigger_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Trigger Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "logs_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Logs", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_delete", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Delete", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_16", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.failed_log !== '[]'", - "fieldname": "log", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Log", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Run", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py deleted file mode 100644 index c734cb105b..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ /dev/null @@ -1,514 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json -import math - -import frappe -from frappe import _ -from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import ( - get_source_value, -) -from frappe.model.document import Document -from frappe.utils import cstr - - -class DataMigrationRun(Document): - @frappe.whitelist() - def run(self): - self.begin() - if self.total_pages > 0: - self.enqueue_next_mapping() - else: - self.complete() - - def enqueue_next_mapping(self): - next_mapping_name = self.get_next_mapping_name() - if next_mapping_name: - next_mapping = self.get_mapping(next_mapping_name) - self.db_set( - dict( - current_mapping=next_mapping.name, - current_mapping_start=0, - current_mapping_delete_start=0, - current_mapping_action="Insert", - ), - notify=True, - commit=True, - ) - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - else: - self.complete() - - def enqueue_next_page(self): - mapping = self.get_mapping(self.current_mapping) - percent_complete = self.percent_complete + (100.0 / self.total_pages) - fields = dict(percent_complete=percent_complete) - if self.current_mapping_action == "Insert": - start = self.current_mapping_start + mapping.page_length - fields["current_mapping_start"] = start - elif self.current_mapping_action == "Delete": - delete_start = self.current_mapping_delete_start + mapping.page_length - fields["current_mapping_delete_start"] = delete_start - - self.db_set(fields, notify=True, commit=True) - - if percent_complete < 100: - frappe.publish_realtime( - self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user - ) - - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - - def run_current_mapping(self): - try: - mapping = self.get_mapping(self.current_mapping) - - if mapping.mapping_type == "Push": - done = self.push() - elif mapping.mapping_type == "Pull": - done = self.pull() - - if done: - self.enqueue_next_mapping() - else: - self.enqueue_next_page() - - except Exception as e: - self.db_set("status", "Error", notify=True, commit=True) - print("Data Migration Run failed") - print(frappe.get_traceback()) - self.execute_postprocess("Error") - raise e - - def get_last_modified_condition(self): - last_run_timestamp = frappe.db.get_value( - "Data Migration Run", - dict( - data_migration_plan=self.data_migration_plan, - data_migration_connector=self.data_migration_connector, - name=("!=", self.name), - ), - "modified", - ) - if last_run_timestamp: - condition = dict(modified=(">", last_run_timestamp)) - else: - condition = {} - return condition - - def begin(self): - plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled] - self.mappings = [ - frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings - ] - - total_pages = 0 - for m in [mapping for mapping in self.mappings]: - if m.mapping_type == "Push": - count = float(self.get_count(m)) - page_count = math.ceil(count / m.page_length) - total_pages += page_count - if m.mapping_type == "Pull": - total_pages += 10 - - self.db_set( - dict( - status="Started", - current_mapping=None, - current_mapping_start=0, - current_mapping_delete_start=0, - percent_complete=0, - current_mapping_action="Insert", - total_pages=total_pages, - ), - notify=True, - commit=True, - ) - - def complete(self): - fields = dict() - - push_failed = self.get_log("push_failed", []) - pull_failed = self.get_log("pull_failed", []) - - status = "Partial Success" - - if not push_failed and not pull_failed: - status = "Success" - fields["percent_complete"] = 100 - - fields["status"] = status - - self.db_set(fields, notify=True, commit=True) - - self.execute_postprocess(status) - - frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user) - - def execute_postprocess(self, status): - # Execute post process - postprocess_method_path = self.get_plan().postprocess_method - - if postprocess_method_path: - frappe.get_attr(postprocess_method_path)( - { - "status": status, - "stats": { - "push_insert": self.push_insert, - "push_update": self.push_update, - "push_delete": self.push_delete, - "pull_insert": self.pull_insert, - "pull_update": self.pull_update, - }, - } - ) - - def get_plan(self): - if not hasattr(self, "plan"): - self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan) - return self.plan - - def get_mapping(self, mapping_name): - if hasattr(self, "mappings"): - for m in self.mappings: - if m.name == mapping_name: - return m - return frappe.get_doc("Data Migration Mapping", mapping_name) - - def get_next_mapping_name(self): - mappings = [m for m in self.get_plan().mappings if m.enabled] - if not self.current_mapping: - # first - return mappings[0].mapping - for i, d in enumerate(mappings): - if i == len(mappings) - 1: - # last - return None - if d.mapping == self.current_mapping: - return mappings[i + 1].mapping - - raise frappe.ValidationError("Mapping Broken") - - def get_data(self, filters): - mapping = self.get_mapping(self.current_mapping) - or_filters = self.get_or_filters(mapping) - start = self.current_mapping_start - - data = [] - doclist = frappe.get_all( - mapping.local_doctype, - filters=filters, - or_filters=or_filters, - start=start, - page_length=mapping.page_length, - ) - - for d in doclist: - doc = frappe.get_doc(mapping.local_doctype, d["name"]) - data.append(doc) - return data - - def get_new_local_data(self): - """Fetch newly inserted local data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # new docs dont have migration field set - filters.update({mapping.migration_id_field: ""}) - - return self.get_data(filters) - - def get_updated_local_data(self): - """Fetch local updated data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # existing docs must have migration field set - filters.update({mapping.migration_id_field: ("!=", "")}) - - return self.get_data(filters) - - def get_deleted_local_data(self): - """Fetch local deleted data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = self.get_last_modified_condition() - filters.update({"deleted_doctype": mapping.local_doctype}) - - data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters) - - _data = [] - for d in data: - doc = json.loads(d.data) - if doc.get(mapping.migration_id_field): - doc["_deleted_document_name"] = d["name"] - _data.append(doc) - - return _data - - def get_remote_data(self): - """Fetch data from remote using `connection.get`. Used during Pull""" - mapping = self.get_mapping(self.current_mapping) - start = self.current_mapping_start - filters = mapping.get_filters() or {} - connection = self.get_connection() - - return connection.get( - mapping.remote_objectname, - fields=["*"], - filters=filters, - start=start, - page_length=mapping.page_length, - ) - - def get_count(self, mapping): - filters = mapping.get_filters() or {} - or_filters = self.get_or_filters(mapping) - - to_insert = frappe.get_all( - mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters - )[0].total - - to_delete = frappe.get_all( - "Deleted Document", - ["count(name) as total"], - filters={"deleted_doctype": mapping.local_doctype}, - or_filters=or_filters, - )[0].total - - return to_insert + to_delete - - def get_or_filters(self, mapping): - or_filters = self.get_last_modified_condition() - - # docs whose migration_id_field is not set - # failed in the previous run, include those too - or_filters.update({mapping.migration_id_field: ("=", "")}) - - return or_filters - - def get_connection(self): - if not hasattr(self, "connection"): - self.connection = frappe.get_doc( - "Data Migration Connector", self.data_migration_connector - ).get_connection() - - return self.connection - - def push(self): - self.db_set("current_mapping_type", "Push") - done = True - - if self.current_mapping_action == "Insert": - done = self._push_insert() - - elif self.current_mapping_action == "Update": - done = self._push_update() - - elif self.current_mapping_action == "Delete": - done = self._push_delete() - - return done - - def _push_insert(self): - """Inserts new local docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_new_local_data() - - for d in data: - # pre process before insert - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - try: - response_doc = connection.insert(mapping.remote_objectname, doc) - frappe.db.set_value( - mapping.local_doctype, - d.name, - mapping.migration_id_field, - response_doc[connection.name_field], - update_modified=False, - ) - frappe.db.commit() - self.update_log("push_insert", 1) - # post process after insert - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to insert - self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_update(self): - """Updates local modified docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_updated_local_data() - - for d in data: - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - try: - response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value) - self.update_log("push_update", 1) - # post process after update - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more data to update - self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_delete(self): - """Deletes docs deleted from local on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_deleted_local_data() - - for d in data: - # Deleted Document also has a custom field for migration_id - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - self.pre_process_doc(d) - try: - response_doc = connection.delete(mapping.remote_objectname, migration_id_value) - self.update_log("push_delete", 1) - # post process only when action is success - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to delete - # done with this mapping - return True - - def pull(self): - self.db_set("current_mapping_type", "Pull") - - connection = self.get_connection() - mapping = self.get_mapping(self.current_mapping) - data = self.get_remote_data() - - for d in data: - migration_id_value = get_source_value(d, connection.name_field) - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - if migration_id_value: - try: - if not local_doc_exists(mapping, migration_id_value): - # insert new local doc - local_doc = insert_local_doc(mapping, doc) - - self.update_log("pull_insert", 1) - # set migration id - frappe.db.set_value( - mapping.local_doctype, - local_doc.name, - mapping.migration_id_field, - migration_id_value, - update_modified=False, - ) - frappe.db.commit() - else: - # update doc - local_doc = update_local_doc(mapping, doc, migration_id_value) - self.update_log("pull_update", 1) - # post process doc after success - self.post_process_doc(remote_doc=d, local_doc=local_doc) - except Exception as e: - # failed, append to log - self.update_log("pull_failed", {migration_id_value: cstr(e)}) - - if len(data) < mapping.page_length: - # last page, done with pull - return True - - def pre_process_doc(self, doc): - plan = self.get_plan() - doc = plan.pre_process_doc(self.current_mapping, doc) - return doc - - def post_process_doc(self, local_doc=None, remote_doc=None): - plan = self.get_plan() - doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc) - return doc - - def set_log(self, key, value): - value = json.dumps(value) if "_failed" in key else value - self.db_set(key, value) - - def update_log(self, key, value=None): - """ - Helper for updating logs, - push_failed and pull_failed are stored as json, - other keys are stored as int - """ - if "_failed" in key: - # json - self.set_log(key, self.get_log(key, []) + [value]) - else: - # int - self.set_log(key, self.get_log(key, 0) + (value or 1)) - - def get_log(self, key, default=None): - value = self.db_get(key) - if "_failed" in key: - if not value: - value = json.dumps(default) - value = json.loads(value) - return value or default - - -def insert_local_doc(mapping, doc): - try: - # insert new doc - if not doc.doctype: - doc.doctype = mapping.local_doctype - doc = frappe.get_doc(doc).insert() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull insert") - print(frappe.get_traceback()) - return None - - -def update_local_doc(mapping, remote_doc, migration_id_value): - try: - # migration id value is set in migration_id_field in mapping.local_doctype - docname = frappe.db.get_value( - mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value} - ) - - doc = frappe.get_doc(mapping.local_doctype, docname) - doc.update(remote_doc) - doc.save() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull update") - print(frappe.get_traceback()) - return None - - -def local_doc_exists(mapping, migration_id_value): - return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value}) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py deleted file mode 100644 index 0357b1e0f5..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - -import frappe - - -class TestDataMigrationRun(unittest.TestCase): - def test_run(self): - create_plan() - - description = "data migration todo" - new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert() - - event_subject = "data migration event" - frappe.get_doc( - dict( - doctype="Event", - subject=event_subject, - repeat_on="Monthly", - starts_on=frappe.utils.now_datetime(), - ) - ).insert() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - self.assertEqual(run.db_get("status"), "Success") - - self.assertEqual(run.db_get("push_insert"), 1) - self.assertEqual(run.db_get("pull_insert"), 1) - - todo = frappe.get_doc("ToDo", new_todo.name) - self.assertTrue(todo.todo_sync_id) - - # Pushed Event - event = frappe.get_doc("Event", todo.todo_sync_id) - self.assertEqual(event.subject, description) - - # Pulled ToDo - created_todo = frappe.get_doc("ToDo", {"description": event_subject}) - self.assertEqual(created_todo.description, event_subject) - - todo_list = frappe.get_list( - "ToDo", filters={"description": "data migration todo"}, fields=["name"] - ) - todo_name = todo_list[0].name - - todo = frappe.get_doc("ToDo", todo_name) - todo.description = "data migration todo updated" - todo.save() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - - # Update - self.assertEqual(run.db_get("status"), "Success") - self.assertEqual(run.db_get("pull_update"), 1) - - -def create_plan(): - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Todo to Event", - "remote_objectname": "Event", - "remote_primary_key": "name", - "mapping_type": "Push", - "local_doctype": "ToDo", - "fields": [ - {"remote_fieldname": "subject", "local_fieldname": "description"}, - { - "remote_fieldname": "starts_on", - "local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())", - }, - ], - "condition": '{"description": "data migration todo" }', - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Event to ToDo", - "remote_objectname": "Event", - "remote_primary_key": "name", - "local_doctype": "ToDo", - "local_primary_key": "name", - "mapping_type": "Pull", - "condition": '{"subject": "data migration event" }', - "fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Plan", - "plan_name": "ToDo Sync", - "module": "Core", - "mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Connector", - "connector_name": "Local Connector", - "connector_type": "Frappe", - # connect to same host. - "hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), - "username": "Administrator", - "password": frappe.conf.get("admin_password") or "admin", - } - ).insert(ignore_if_duplicate=True) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 4c535b2811..93b883cda6 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -53,22 +53,6 @@ def sync_for(app_name, force=0, reset_permissions=False): os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json") ) - for data_migration_module in [ - "data_migration_mapping_detail", - "data_migration_mapping", - "data_migration_plan_mapping", - "data_migration_plan", - ]: - files.append( - os.path.join( - FRAPPE_PATH, - "data_migration", - "doctype", - data_migration_module, - f"{data_migration_module}.json", - ) - ) - for desk_module in [ "number_card", "dashboard_chart", @@ -124,8 +108,6 @@ def get_doc_files(files, start_path): "web_template", "notification", "print_style", - "data_migration_mapping", - "data_migration_plan", "workspace", "onboarding_step", "module_onboarding", diff --git a/frappe/modules.txt b/frappe/modules.txt index a707ca853e..fb7817f6ba 100644 --- a/frappe/modules.txt +++ b/frappe/modules.txt @@ -8,7 +8,6 @@ Desk Integrations Printing Contacts -Data Migration Social Automation Event Streaming \ No newline at end of file diff --git a/frappe/patches.txt b/frappe/patches.txt index 6c46d5dcd9..b771485c3c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -201,3 +201,4 @@ frappe.patches.v14_0.update_color_names_in_kanban_board_column frappe.patches.v14_0.update_is_system_generated_flag frappe.patches.v14_0.update_auto_account_deletion_duration frappe.patches.v14_0.set_document_expiry_default +frappe.patches.v14_0.delete_data_migration_tool \ No newline at end of file diff --git a/frappe/patches/v14_0/delete_data_migration_tool.py b/frappe/patches/v14_0/delete_data_migration_tool.py new file mode 100644 index 0000000000..d0416cb1e7 --- /dev/null +++ b/frappe/patches/v14_0/delete_data_migration_tool.py @@ -0,0 +1,12 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe + + +def execute(): + doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name") + for doctype in doctypes: + frappe.delete_doc("DocType", doctype, ignore_missing=True) + + frappe.delete_doc("Module Def", "Data Migration", ignore_missing=True, force=True) From 8c8249894beb0b6707954485e2b3947192e8025b Mon Sep 17 00:00:00 2001 From: Abhirup Pal Date: Mon, 13 Jun 2022 16:41:43 +0530 Subject: [PATCH 10/21] fix: Update label for editing in full page and add custom logic for rendering it. (#17149) --- cypress/integration/timeline_email.js | 2 +- frappe/public/js/frappe/form/quick_entry.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cypress/integration/timeline_email.js b/cypress/integration/timeline_email.js index f2a239401d..993847bcb8 100644 --- a/cypress/integration/timeline_email.js +++ b/cypress/integration/timeline_email.js @@ -7,7 +7,7 @@ context('Timeline Email', () => { it('Adding new ToDo', () => { cy.click_listview_primary_button('Add ToDo'); - cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500}); + cy.get('.custom-actions:visible > .btn').contains("Edit Full Form").click({delay: 500}); cy.fill_field("description", "Test ToDo", "Text Editor"); cy.wait(500); cy.get('.primary-action').contains('Save').click({force: true}); diff --git a/frappe/public/js/frappe/form/quick_entry.js b/frappe/public/js/frappe/form/quick_entry.js index 86523d7088..f55f7139c9 100644 --- a/frappe/public/js/frappe/form/quick_entry.js +++ b/frappe/public/js/frappe/form/quick_entry.js @@ -267,7 +267,7 @@ frappe.ui.form.QuickEntryForm = class QuickEntryForm { render_edit_in_full_page_link() { var me = this; this.dialog.add_custom_action( - `${__("Edit in full page")}`, + `${__("Edit Full Form")}`, () => me.open_doc(true) ); } From cfc4d966c7113683b1e96cd08df801194312bc05 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:03:14 +0530 Subject: [PATCH 11/21] fix(ux): patch title field and make patch read only --- frappe/core/doctype/patch_log/patch_log.json | 115 ++++++------------- 1 file changed, 36 insertions(+), 79 deletions(-) diff --git a/frappe/core/doctype/patch_log/patch_log.json b/frappe/core/doctype/patch_log/patch_log.json index aa054f1360..9750c51279 100644 --- a/frappe/core/doctype/patch_log/patch_log.json +++ b/frappe/core/doctype/patch_log/patch_log.json @@ -1,87 +1,44 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "PATCHLOG.#####", - "beta": 0, - "creation": "2013-01-17 11:36:45", - "custom": 0, - "description": "List of patches executed", - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, + "actions": [], + "autoname": "PATCHLOG.#####", + "creation": "2013-01-17 11:36:45", + "description": "List of patches executed", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "patch" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "patch", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Patch", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "patch", + "fieldtype": "Code", + "label": "Patch", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-cog", - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:40:35.048570", - "modified_by": "Administrator", - "module": "Core", - "name": "Patch Log", - "owner": "Administrator", + ], + "icon": "fa fa-cog", + "idx": 1, + "links": [], + "modified": "2022-06-13 05:34:37.845368", + "modified_by": "Administrator", + "module": "Core", + "name": "Patch Log", + "naming_rule": "Expression (old style)", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator" } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "patch", + "track_changes": 1 } \ No newline at end of file From 602e8d0a7863608d0c5d11267c0bec72cdc76230 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:11:35 +0530 Subject: [PATCH 12/21] fix(ux): show title field on scheduled job log --- .../core/doctype/scheduled_job_log/scheduled_job_log.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json index 396b32bdf9..451c4108a0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json @@ -38,7 +38,7 @@ } ], "links": [], - "modified": "2021-10-25 00:00:00.000000", + "modified": "2022-06-13 05:41:21.090972", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Log", @@ -59,5 +59,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "DESC" -} + "sort_order": "DESC", + "states": [], + "title_field": "scheduled_job_type" +} \ No newline at end of file From 3730e20618a26887d5c1a4f7819c6b27c0d0bcfa Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:15:30 +0530 Subject: [PATCH 13/21] fix(ux): better layout on error log - plus show button to see relevant error logs --- frappe/core/doctype/error_log/error_log.js | 15 +++++++++++---- frappe/core/doctype/error_log/error_log.json | 18 ++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 4fe8fde5d6..277969edd9 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -1,8 +1,15 @@ -// Copyright (c) 2016, Frappe Technologies and contributors +// Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Error Log', { +frappe.ui.form.on("Error Log", { refresh: function(frm) { - - } + if (frm.doc.reference_doctype && frm.doc.reference_name) { + frm.add_custom_button(__("Show Errors for This Document"), function() { + frappe.set_route("List", "Error Log", { + reference_doctype: frm.doc.reference_doctype, + reference_name: frm.doc.reference_name, + }); + }); + } + }, }); diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index e0ce109595..a93161c891 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -6,10 +6,12 @@ "engine": "MyISAM", "field_order": [ "seen", - "method", - "error", "reference_doctype", - "reference_name" + "column_break_3", + "reference_name", + "section_break_5", + "method", + "error" ], "fields": [ { @@ -47,12 +49,20 @@ "fieldtype": "Data", "label": "Reference Name", "read_only": 1 + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_5", + "fieldtype": "Section Break" } ], "icon": "fa fa-warning-sign", "idx": 1, "links": [], - "modified": "2022-05-19 05:32:16.026684", + "modified": "2022-06-13 05:43:30.969901", "modified_by": "Administrator", "module": "Core", "name": "Error Log", From 1de9c6ac68ea5f29e4144b7a879f6667ff668e8a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:18:25 +0530 Subject: [PATCH 14/21] fix(ux): add user as filter on route history --- .../doctype/route_history/route_history.json | 100 +++--------------- 1 file changed, 13 insertions(+), 87 deletions(-) diff --git a/frappe/desk/doctype/route_history/route_history.json b/frappe/desk/doctype/route_history/route_history.json index 09db2320ca..a5d73fc360 100644 --- a/frappe/desk/doctype/route_history/route_history.json +++ b/frappe/desk/doctype/route_history/route_history.json @@ -1,126 +1,52 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-10-05 11:26:04.601113", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "route", + "user" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "route", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, - "label": "Route", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Route" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "user", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, + "in_standard_filter": 1, "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "User" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 13:26:03.106050", + "links": [], + "modified": "2022-06-13 05:48:56.967244", "modified_by": "Administrator", "module": "Desk", "name": "Route History", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_seen": 0, - "track_views": 0 -} + "states": [], + "title_field": "route" +} \ No newline at end of file From 149594448ead89608eb46971cbca3d9fbbf639db Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:21:53 +0530 Subject: [PATCH 15/21] fix(ux): make deleted doctype read only - also add filter for doctype in list view --- .../deleted_document/deleted_document.json | 309 ++++-------------- 1 file changed, 67 insertions(+), 242 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document.json b/frappe/core/doctype/deleted_document/deleted_document.json index 1a612c7411..6b95a523c1 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.json +++ b/frappe/core/doctype/deleted_document/deleted_document.json @@ -1,256 +1,81 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-12-29 12:59:48.638970", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2016-12-29 12:59:48.638970", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "deleted_name", + "deleted_doctype", + "column_break_3", + "restored", + "new_name", + "section_break_6", + "data" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_name", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted Name", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_doctype", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted DocType", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_doctype", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted DocType", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restored", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Restored", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "restored", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Restored", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "new_name", - "fieldtype": "Read Only", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "New Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "new_name", + "fieldtype": "Read Only", + "label": "New Name" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_6", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_6", + "fieldtype": "Section Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "data", + "fieldtype": "Code", + "label": "Data", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:39:45.724494", - "modified_by": "Administrator", - "module": "Core", - "name": "Deleted Document", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-06-13 05:50:58.314908", + "modified_by": "Administrator", + "module": "Core", + "name": "Deleted Document", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "delete": 1, + "export": 1, + "read": 1, + "role": "System Manager" } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "deleted_name", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "deleted_name", + "track_changes": 1 } \ No newline at end of file From 37086c035c4f1130ceba591a3b6a27778e673947 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:29:46 +0530 Subject: [PATCH 16/21] fix: show user and docname filter on access log --- frappe/core/doctype/access_log/access_log.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/access_log/access_log.json b/frappe/core/doctype/access_log/access_log.json index c5f1030266..69803ef05a 100644 --- a/frappe/core/doctype/access_log/access_log.json +++ b/frappe/core/doctype/access_log/access_log.json @@ -36,6 +36,7 @@ "fieldname": "user", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "User ", "options": "User", "read_only": 1 @@ -51,6 +52,7 @@ "fieldname": "reference_document", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document", "read_only": 1 }, @@ -129,7 +131,7 @@ } ], "links": [], - "modified": "2022-05-03 09:34:19.337551", + "modified": "2022-06-13 05:59:26.866004", "modified_by": "Administrator", "module": "Core", "name": "Access Log", From ec860cacea07a1ecb6b9042769c2a8a9425b76bf Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:34:39 +0530 Subject: [PATCH 17/21] fix(ux): list view filters for server script --- frappe/core/doctype/server_script/server_script.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 9312ae178b..5446cc1a39 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -25,6 +25,7 @@ "fieldname": "script_type", "fieldtype": "Select", "in_list_view": 1, + "in_standard_filter": 1, "label": "Script Type", "options": "DocType Event\nScheduler Event\nPermission Query\nAPI", "reqd": 1 @@ -41,6 +42,7 @@ "fieldname": "reference_doctype", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document Type", "options": "DocType" }, @@ -109,7 +111,7 @@ "link_fieldname": "server_script" } ], - "modified": "2022-04-27 11:42:52.032963", + "modified": "2022-06-13 06:04:20.937969", "modified_by": "Administrator", "module": "Core", "name": "Server Script", From 77d03d239387e562c9f09c46efc4cc37d95400b4 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:44:51 +0530 Subject: [PATCH 18/21] fix: add list view filters for prepared report --- frappe/core/doctype/prepared_report/prepared_report.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index 4663dcb463..114b5f5b13 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -31,6 +31,7 @@ "fieldname": "ref_report_doctype", "fieldtype": "Link", "hidden": 1, + "in_standard_filter": 1, "label": "Ref Report DocType", "options": "Report", "read_only": 1 @@ -41,6 +42,7 @@ "fieldtype": "Select", "hidden": 1, "in_list_view": 1, + "in_standard_filter": 1, "label": "Status", "options": "Error\nQueued\nCompleted", "read_only": 1 @@ -103,10 +105,11 @@ ], "in_create": 1, "links": [], - "modified": "2020-03-05 10:52:56.598365", + "modified": "2022-06-13 06:14:30.144230", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", + "naming_rule": "Expression (old style)", "owner": "Administrator", "permissions": [ { @@ -134,6 +137,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "report_name", "track_changes": 1 } \ No newline at end of file From 9b7b32e60417bcfd13749aac51825082ccbb10c8 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:52:07 +0530 Subject: [PATCH 19/21] fix(ux): report and status filter for prepared_report --- frappe/core/doctype/prepared_report/prepared_report.json | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index 114b5f5b13..cafe323519 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -23,16 +23,14 @@ { "fieldname": "report_name", "fieldtype": "Data", - "hidden": 1, "label": "Report Name", "read_only": 1 }, { "fieldname": "ref_report_doctype", "fieldtype": "Link", - "hidden": 1, "in_standard_filter": 1, - "label": "Ref Report DocType", + "label": "Report Type", "options": "Report", "read_only": 1 }, @@ -105,7 +103,7 @@ ], "in_create": 1, "links": [], - "modified": "2022-06-13 06:14:30.144230", + "modified": "2022-06-13 06:20:34.496412", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", @@ -134,10 +132,9 @@ "share": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", "states": [], - "title_field": "report_name", + "title_field": "ref_report_doctype", "track_changes": 1 } \ No newline at end of file From cc0672537ea8815326b49d4802c31a445f8b0697 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 16:05:28 +0530 Subject: [PATCH 20/21] fix: make error/patch log read only --- frappe/core/doctype/error_log/error_log.js | 4 +++- frappe/core/doctype/error_log/error_log.json | 4 ++-- frappe/core/doctype/patch_log/patch_log.js | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 277969edd9..1262002b04 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -3,8 +3,10 @@ frappe.ui.form.on("Error Log", { refresh: function(frm) { + frm.disable_save(); + if (frm.doc.reference_doctype && frm.doc.reference_name) { - frm.add_custom_button(__("Show Errors for This Document"), function() { + frm.add_custom_button(__("Show Related Errors"), function() { frappe.set_route("List", "Error Log", { reference_doctype: frm.doc.reference_doctype, reference_name: frm.doc.reference_name, diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index a93161c891..2ee86bd118 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -61,8 +61,9 @@ ], "icon": "fa fa-warning-sign", "idx": 1, + "in_create": 1, "links": [], - "modified": "2022-06-13 05:43:30.969901", + "modified": "2022-06-13 06:34:05.158606", "modified_by": "Administrator", "module": "Core", "name": "Error Log", @@ -80,7 +81,6 @@ "write": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "ASC", "states": [], diff --git a/frappe/core/doctype/patch_log/patch_log.js b/frappe/core/doctype/patch_log/patch_log.js index 0080584a29..b52876ac97 100644 --- a/frappe/core/doctype/patch_log/patch_log.js +++ b/frappe/core/doctype/patch_log/patch_log.js @@ -3,6 +3,6 @@ frappe.ui.form.on('Patch Log', { refresh: function(frm) { - + frm.disable_save(); } }); From 6dfbf7b19ea735d45a61c3e3a3e70be540827b60 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 16:10:36 +0530 Subject: [PATCH 21/21] fix: show doctype and fieldname filters --- frappe/custom/doctype/custom_field/custom_field.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index 045a0981f3..63be70c644 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -67,7 +67,8 @@ "fieldtype": "Link", "in_filter": 1, "in_list_view": 1, - "label": "Document", + "in_standard_filter": 1, + "label": "DocType", "oldfieldname": "dt", "oldfieldtype": "Link", "options": "DocType", @@ -94,6 +95,7 @@ "fieldname": "fieldname", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Fieldname", "no_copy": 1, "oldfieldname": "fieldname", @@ -439,7 +441,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-04-14 09:46:58.849765", + "modified": "2022-06-13 06:39:03.319667", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field",