diff --git a/cypress/integration/timeline_email.js b/cypress/integration/timeline_email.js index f2a239401d..993847bcb8 100644 --- a/cypress/integration/timeline_email.js +++ b/cypress/integration/timeline_email.js @@ -7,7 +7,7 @@ context('Timeline Email', () => { it('Adding new ToDo', () => { cy.click_listview_primary_button('Add ToDo'); - cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500}); + cy.get('.custom-actions:visible > .btn').contains("Edit Full Form").click({delay: 500}); cy.fill_field("description", "Test ToDo", "Text Editor"); cy.wait(500); cy.get('.primary-action').contains('Save').click({force: true}); diff --git a/frappe/core/doctype/access_log/access_log.json b/frappe/core/doctype/access_log/access_log.json index c5f1030266..69803ef05a 100644 --- a/frappe/core/doctype/access_log/access_log.json +++ b/frappe/core/doctype/access_log/access_log.json @@ -36,6 +36,7 @@ "fieldname": "user", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "User ", "options": "User", "read_only": 1 @@ -51,6 +52,7 @@ "fieldname": "reference_document", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document", "read_only": 1 }, @@ -129,7 +131,7 @@ } ], "links": [], - "modified": "2022-05-03 09:34:19.337551", + "modified": "2022-06-13 05:59:26.866004", "modified_by": "Administrator", "module": "Core", "name": "Access Log", diff --git a/frappe/core/doctype/deleted_document/deleted_document.json b/frappe/core/doctype/deleted_document/deleted_document.json index 1a612c7411..6b95a523c1 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.json +++ b/frappe/core/doctype/deleted_document/deleted_document.json @@ -1,256 +1,81 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-12-29 12:59:48.638970", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2016-12-29 12:59:48.638970", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "deleted_name", + "deleted_doctype", + "column_break_3", + "restored", + "new_name", + "section_break_6", + "data" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_name", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted Name", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_doctype", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted DocType", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_doctype", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted DocType", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restored", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Restored", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "restored", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Restored", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "new_name", - "fieldtype": "Read Only", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "New Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "new_name", + "fieldtype": "Read Only", + "label": "New Name" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_6", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_6", + "fieldtype": "Section Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "data", + "fieldtype": "Code", + "label": "Data", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:39:45.724494", - "modified_by": "Administrator", - "module": "Core", - "name": "Deleted Document", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-06-13 05:50:58.314908", + "modified_by": "Administrator", + "module": "Core", + "name": "Deleted Document", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "delete": 1, + "export": 1, + "read": 1, + "role": "System Manager" } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "deleted_name", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "deleted_name", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 4fe8fde5d6..1262002b04 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -1,8 +1,17 @@ -// Copyright (c) 2016, Frappe Technologies and contributors +// Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Error Log', { +frappe.ui.form.on("Error Log", { refresh: function(frm) { + frm.disable_save(); - } + if (frm.doc.reference_doctype && frm.doc.reference_name) { + frm.add_custom_button(__("Show Related Errors"), function() { + frappe.set_route("List", "Error Log", { + reference_doctype: frm.doc.reference_doctype, + reference_name: frm.doc.reference_name, + }); + }); + } + }, }); diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index e0ce109595..2ee86bd118 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -6,10 +6,12 @@ "engine": "MyISAM", "field_order": [ "seen", - "method", - "error", "reference_doctype", - "reference_name" + "column_break_3", + "reference_name", + "section_break_5", + "method", + "error" ], "fields": [ { @@ -47,12 +49,21 @@ "fieldtype": "Data", "label": "Reference Name", "read_only": 1 + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_5", + "fieldtype": "Section Break" } ], "icon": "fa fa-warning-sign", "idx": 1, + "in_create": 1, "links": [], - "modified": "2022-05-19 05:32:16.026684", + "modified": "2022-06-13 06:34:05.158606", "modified_by": "Administrator", "module": "Core", "name": "Error Log", @@ -70,7 +81,6 @@ "write": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "ASC", "states": [], diff --git a/frappe/core/doctype/patch_log/patch_log.js b/frappe/core/doctype/patch_log/patch_log.js index 0080584a29..b52876ac97 100644 --- a/frappe/core/doctype/patch_log/patch_log.js +++ b/frappe/core/doctype/patch_log/patch_log.js @@ -3,6 +3,6 @@ frappe.ui.form.on('Patch Log', { refresh: function(frm) { - + frm.disable_save(); } }); diff --git a/frappe/core/doctype/patch_log/patch_log.json b/frappe/core/doctype/patch_log/patch_log.json index aa054f1360..9750c51279 100644 --- a/frappe/core/doctype/patch_log/patch_log.json +++ b/frappe/core/doctype/patch_log/patch_log.json @@ -1,87 +1,44 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "PATCHLOG.#####", - "beta": 0, - "creation": "2013-01-17 11:36:45", - "custom": 0, - "description": "List of patches executed", - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, + "actions": [], + "autoname": "PATCHLOG.#####", + "creation": "2013-01-17 11:36:45", + "description": "List of patches executed", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "patch" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "patch", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Patch", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "patch", + "fieldtype": "Code", + "label": "Patch", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-cog", - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:40:35.048570", - "modified_by": "Administrator", - "module": "Core", - "name": "Patch Log", - "owner": "Administrator", + ], + "icon": "fa fa-cog", + "idx": 1, + "links": [], + "modified": "2022-06-13 05:34:37.845368", + "modified_by": "Administrator", + "module": "Core", + "name": "Patch Log", + "naming_rule": "Expression (old style)", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator" } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "patch", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index 4663dcb463..cafe323519 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -23,15 +23,14 @@ { "fieldname": "report_name", "fieldtype": "Data", - "hidden": 1, "label": "Report Name", "read_only": 1 }, { "fieldname": "ref_report_doctype", "fieldtype": "Link", - "hidden": 1, - "label": "Ref Report DocType", + "in_standard_filter": 1, + "label": "Report Type", "options": "Report", "read_only": 1 }, @@ -41,6 +40,7 @@ "fieldtype": "Select", "hidden": 1, "in_list_view": 1, + "in_standard_filter": 1, "label": "Status", "options": "Error\nQueued\nCompleted", "read_only": 1 @@ -103,10 +103,11 @@ ], "in_create": 1, "links": [], - "modified": "2020-03-05 10:52:56.598365", + "modified": "2022-06-13 06:20:34.496412", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", + "naming_rule": "Expression (old style)", "owner": "Administrator", "permissions": [ { @@ -131,9 +132,9 @@ "share": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", - "title_field": "report_name", + "states": [], + "title_field": "ref_report_doctype", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json index 396b32bdf9..451c4108a0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json @@ -38,7 +38,7 @@ } ], "links": [], - "modified": "2021-10-25 00:00:00.000000", + "modified": "2022-06-13 05:41:21.090972", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Log", @@ -59,5 +59,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "DESC" -} + "sort_order": "DESC", + "states": [], + "title_field": "scheduled_job_type" +} \ No newline at end of file diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 9312ae178b..5446cc1a39 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -25,6 +25,7 @@ "fieldname": "script_type", "fieldtype": "Select", "in_list_view": 1, + "in_standard_filter": 1, "label": "Script Type", "options": "DocType Event\nScheduler Event\nPermission Query\nAPI", "reqd": 1 @@ -41,6 +42,7 @@ "fieldname": "reference_doctype", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document Type", "options": "DocType" }, @@ -109,7 +111,7 @@ "link_fieldname": "server_script" } ], - "modified": "2022-04-27 11:42:52.032963", + "modified": "2022-06-13 06:04:20.937969", "modified_by": "Administrator", "module": "Core", "name": "Server Script", diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index 045a0981f3..63be70c644 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -67,7 +67,8 @@ "fieldtype": "Link", "in_filter": 1, "in_list_view": 1, - "label": "Document", + "in_standard_filter": 1, + "label": "DocType", "oldfieldname": "dt", "oldfieldtype": "Link", "options": "DocType", @@ -94,6 +95,7 @@ "fieldname": "fieldname", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Fieldname", "no_copy": 1, "oldfieldname": "fieldname", @@ -439,7 +441,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-04-14 09:46:58.849765", + "modified": "2022-06-13 06:39:03.319667", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field", diff --git a/frappe/data_migration/__init__.py b/frappe/data_migration/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/__init__.py b/frappe/data_migration/doctype/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/__init__.py b/frappe/data_migration/doctype/data_migration_connector/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py b/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py deleted file mode 100644 index 7d2b320c59..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py +++ /dev/null @@ -1,24 +0,0 @@ -from abc import ABCMeta, abstractmethod - -from frappe.utils.password import get_decrypted_password - - -class BaseConnection(metaclass=ABCMeta): - @abstractmethod - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - @abstractmethod - def insert(self, doctype, doc): - pass - - @abstractmethod - def update(self, doctype, doc, migration_id): - pass - - @abstractmethod - def delete(self, doctype, migration_id): - pass - - def get_password(self): - return get_decrypted_password("Data Migration Connector", self.connector.name) diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py deleted file mode 100644 index 8228529562..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py +++ /dev/null @@ -1,32 +0,0 @@ -import frappe -from frappe.frappeclient import FrappeClient - -from .base import BaseConnection - - -class FrappeConnection(BaseConnection): - def __init__(self, connector): - self.connector = connector - self.connection = FrappeClient( - self.connector.hostname, self.connector.username, self.get_password() - ) - self.name_field = "name" - - def insert(self, doctype, doc): - doc = frappe._dict(doc) - doc.doctype = doctype - return self.connection.insert(doc) - - def update(self, doctype, doc, migration_id): - doc = frappe._dict(doc) - doc.doctype = doctype - doc.name = migration_id - return self.connection.update(doc) - - def delete(self, doctype, migration_id): - return self.connection.delete(doctype, migration_id) - - def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20): - return self.connection.get_list( - doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length - ) diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js deleted file mode 100644 index 0898fcf4e7..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Connector', { - onload(frm) { - if(frappe.boot.developer_mode) { - frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm)); - } - }, - new_connection(frm) { - const d = new frappe.ui.Dialog({ - title: __('New Connection'), - fields: [ - { label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 }, - { label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 }, - ], - primary_action_label: __('Create'), - primary_action: (values) => { - let { module, connection_name } = values; - - frm.events.create_new_connection(module, connection_name) - .then(r => { - if (r.message) { - const connector_name = connection_name - .replace('connection', 'Connector') - .replace('Connection', 'Connector') - .trim(); - - frm.set_value('connector_name', connector_name); - frm.set_value('connector_type', 'Custom'); - frm.set_value('python_module', r.message); - frm.save(); - frappe.show_alert(__("New module created {0}", [r.message])); - d.hide(); - } - }); - } - }); - - d.show(); - }, - create_new_connection(module, connection_name) { - return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', { - module, connection_name - }); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json deleted file mode 100644 index 338d59aadd..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:connector_name", - "beta": 1, - "creation": "2017-08-11 05:03:27.091416", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "connector_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:!doc.is_custom", - "fieldname": "connector_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Type", - "length": 0, - "no_copy": 0, - "options": "\nFrappe\nCustom", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.connector_type == 'Custom'", - "fieldname": "python_module", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Python Module", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "authentication_credentials", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Authentication Credentials", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "", - "fieldname": "hostname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Hostname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "database_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Database Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "username", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Username", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "password", - "fieldtype": "Password", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Password", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-12-01 13:38:55.992499", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Connector", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py deleted file mode 100644 index 9db7fc2445..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import os - -import frappe -from frappe import _ -from frappe.model.document import Document -from frappe.modules.export_file import create_init_py - -from .connectors.base import BaseConnection -from .connectors.frappe_connection import FrappeConnection - - -class DataMigrationConnector(Document): - def validate(self): - if not (self.python_module or self.connector_type): - frappe.throw(_("Enter python module or select connector type")) - - if self.python_module: - try: - get_connection_class(self.python_module) - except: - frappe.throw(frappe._("Invalid module path")) - - def get_connection(self): - if self.python_module: - _class = get_connection_class(self.python_module) - return _class(self) - else: - self.connection = FrappeConnection(self) - - return self.connection - - -@frappe.whitelist() -def create_new_connection(module, connection_name): - if not frappe.conf.get("developer_mode"): - frappe.msgprint(_("Please enable developer mode to create new connection")) - return - # create folder - module_path = frappe.get_module_path(module) - connectors_folder = os.path.join(module_path, "connectors") - frappe.create_folder(connectors_folder) - - # create init py - create_init_py(module_path, "connectors", "") - - connection_class = connection_name.replace(" ", "") - file_name = frappe.scrub(connection_name) + ".py" - file_path = os.path.join(module_path, "connectors", file_name) - - # create boilerplate file - with open(file_path, "w") as f: - f.write(connection_boilerplate.format(connection_class=connection_class)) - - # get python module string from file_path - app_name = frappe.db.get_value("Module Def", module, "app_name") - python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace( - os.path.sep, "." - )[:-3] - - return python_module - - -def get_connection_class(python_module): - filename = python_module.rsplit(".", 1)[-1] - classname = frappe.unscrub(filename).replace(" ", "") - module = frappe.get_module(python_module) - - raise_error = False - if hasattr(module, classname): - _class = getattr(module, classname) - if not issubclass(_class, BaseConnection): - raise_error = True - else: - raise_error = True - - if raise_error: - raise ImportError(filename) - - return _class - - -connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection - -class {connection_class}(BaseConnection): - def __init__(self, connector): - # self.connector = connector - # self.connection = YourModule(self.connector.username, self.get_password()) - # self.name_field = 'id' - pass - - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - def insert(self, doctype, doc): - pass - - def update(self, doctype, doc, migration_id): - pass - - def delete(self, doctype, migration_id): - pass - -""" diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py deleted file mode 100644 index c4090796ab..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationConnector(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js deleted file mode 100644 index 6c99b9a54d..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Mapping', { - refresh: function() { - - } -}); diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json deleted file mode 100644 index 998abdf6ca..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json +++ /dev/null @@ -1,456 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:mapping_name", - "beta": 1, - "creation": "2017-08-11 05:11:49.975801", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_objectname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Objectname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local DocType", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Local Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_5", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull\nSync", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "10", - "fieldname": "page_length", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page Length", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "migration_id_field", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Migration ID Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "fields", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Field Maps", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping Detail", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 1, - "columns": 0, - "fieldname": "condition_detail", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition Detail", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "condition", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-09-27 18:06:43.275207", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py deleted file mode 100644 index 49af65e99b..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.model.document import Document -from frappe.utils.safe_exec import get_safe_globals - - -class DataMigrationMapping(Document): - def get_filters(self): - if self.condition: - return frappe.safe_eval(self.condition, get_safe_globals()) - - def get_fields(self): - fields = [] - for f in self.fields: - if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")): - fields.append(f.local_fieldname) - - if frappe.db.has_column(self.local_doctype, self.migration_id_field): - fields.append(self.migration_id_field) - - if "name" not in fields: - fields.append("name") - - return fields - - def get_mapped_record(self, doc): - """Build a mapped record using information from the fields table""" - mapped = frappe._dict() - - key_fieldname = "remote_fieldname" - value_fieldname = "local_fieldname" - - if self.mapping_type == "Pull": - key_fieldname, value_fieldname = value_fieldname, key_fieldname - - for field_map in self.fields: - key = get_source_value(field_map, key_fieldname) - - if not field_map.is_child_table: - # field to field mapping - value = get_value_from_fieldname(field_map, value_fieldname, doc) - else: - # child table mapping - mapping_name = field_map.child_table_mapping - value = get_mapped_child_records( - mapping_name, doc.get(get_source_value(field_map, value_fieldname)) - ) - - mapped[key] = value - - return mapped - - -def get_mapped_child_records(mapping_name, child_docs): - mapped_child_docs = [] - mapping = frappe.get_doc("Data Migration Mapping", mapping_name) - for child_doc in child_docs: - mapped_child_docs.append(mapping.get_mapped_record(child_doc)) - - return mapped_child_docs - - -def get_value_from_fieldname(field_map, fieldname_field, doc): - field_name = get_source_value(field_map, fieldname_field) - - if field_name.startswith("eval:"): - value = frappe.safe_eval(field_name[5:], get_safe_globals()) - elif field_name[0] in ('"', "'"): - value = field_name[1:-1] - else: - value = get_source_value(doc, field_name) - return value - - -def get_source_value(source, key): - """Get value from source (object or dict) based on key""" - if isinstance(source, dict): - return source.get(key) - else: - return getattr(source, key) diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py deleted file mode 100644 index 30d2a6bcfe..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationMapping(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py b/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json deleted file mode 100644 index ede9213f14..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-08-11 05:09:10.900237", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "is_child_table", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Is Child Table", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "is_child_table", - "fieldname": "child_table_mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Child Table Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-28 17:13:31.337005", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping Detail", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py deleted file mode 100644 index abd6348a26..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationMappingDetail(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan/__init__.py b/frappe/data_migration/doctype/data_migration_plan/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js deleted file mode 100644 index 357ef2972f..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Plan', { - onload(frm) { - frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', { - data_migration_plan: frm.doc.name - })); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json deleted file mode 100644 index 2cfc2e3bd7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:plan_name", - "beta": 0, - "creation": "2017-08-11 05:15:51.482165", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "plan_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Plan Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 1 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Module", - "length": 0, - "no_copy": 0, - "options": "Module Def", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mappings", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mappings", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "preprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Preprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "postprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Postprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py deleted file mode 100644 index 4118e8e7fe..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.model.document import Document -from frappe.modules import get_module_path, scrub_dt_dn -from frappe.modules.export_file import create_init_py, export_to_files - - -def get_mapping_module(module, mapping_name): - app_name = frappe.db.get_value("Module Def", module, "app_name") - mapping_name = frappe.scrub(mapping_name) - module = frappe.scrub(module) - - try: - return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}") - except ImportError: - return None - - -class DataMigrationPlan(Document): - def on_update(self): - # update custom fields in mappings - self.make_custom_fields_for_mappings() - - if frappe.flags.in_import or frappe.flags.in_test: - return - - if frappe.local.conf.get("developer_mode"): - record_list = [["Data Migration Plan", self.name]] - - for m in self.mappings: - record_list.append(["Data Migration Mapping", m.mapping]) - - export_to_files(record_list=record_list, record_module=self.module) - - for m in self.mappings: - dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping) - create_init_py(get_module_path(self.module), dt, dn) - - def make_custom_fields_for_mappings(self): - frappe.flags.ignore_in_install = True - label = self.name + " ID" - fieldname = frappe.scrub(label) - - df = { - "label": label, - "fieldname": fieldname, - "fieldtype": "Data", - "hidden": 1, - "read_only": 1, - "unique": 1, - "no_copy": 1, - } - - for m in self.mappings: - mapping = frappe.get_doc("Data Migration Mapping", m.mapping) - create_custom_field(mapping.local_doctype, df) - mapping.migration_id_field = fieldname - mapping.save() - - # Create custom field in Deleted Document - create_custom_field("Deleted Document", df) - frappe.flags.ignore_in_install = False - - def pre_process_doc(self, mapping_name, doc): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "pre_process"): - return module.pre_process(doc) - return doc - - def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "post_process"): - return module.post_process(local_doc=local_doc, remote_doc=remote_doc) diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py deleted file mode 100644 index ef3bfa3a70..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationPlan(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json deleted file mode 100644 index 5acf014715..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 1, - "creation": "2017-08-11 05:15:38.390831", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "enabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Enabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-20 21:43:04.908650", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py deleted file mode 100644 index 0650f4b2c7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationPlanMapping(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_run/__init__.py b/frappe/data_migration/doctype/data_migration_run/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js b/frappe/data_migration/doctype/data_migration_run/data_migration_run.js deleted file mode 100644 index 82323c62f1..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Run', { - refresh: function(frm) { - if (frm.doc.status !== 'Success') { - frm.add_custom_button(__('Run'), () => frm.call('run')); - } - if (frm.doc.status === 'Started') { - frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete, - __('Currently updating {0}', [frm.doc.current_mapping])); - } - } -}); diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json b/frappe/data_migration/doctype/data_migration_run/data_migration_run.json deleted file mode 100644 index db77997928..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json +++ /dev/null @@ -1,838 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-09-11 12:55:27.597728", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_plan", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Data Migration Plan", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_connector", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data Migration Connector", - "length": 0, - "no_copy": 0, - "options": "Data Migration Connector", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 1, - "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "start_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Start Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "end_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "End Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Remote ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping", - "length": 0, - "no_copy": 1, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_delete_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Delete Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "current_mapping_action", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Action", - "length": 0, - "no_copy": 1, - "options": "Insert\nDelete", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "total_pages", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Total Pages", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "percent_complete", - "fieldtype": "Percent", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Percent Complete", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "trigger_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Trigger Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "logs_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Logs", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_delete", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Delete", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_16", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.failed_log !== '[]'", - "fieldname": "log", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Log", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Run", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py deleted file mode 100644 index c734cb105b..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ /dev/null @@ -1,514 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json -import math - -import frappe -from frappe import _ -from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import ( - get_source_value, -) -from frappe.model.document import Document -from frappe.utils import cstr - - -class DataMigrationRun(Document): - @frappe.whitelist() - def run(self): - self.begin() - if self.total_pages > 0: - self.enqueue_next_mapping() - else: - self.complete() - - def enqueue_next_mapping(self): - next_mapping_name = self.get_next_mapping_name() - if next_mapping_name: - next_mapping = self.get_mapping(next_mapping_name) - self.db_set( - dict( - current_mapping=next_mapping.name, - current_mapping_start=0, - current_mapping_delete_start=0, - current_mapping_action="Insert", - ), - notify=True, - commit=True, - ) - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - else: - self.complete() - - def enqueue_next_page(self): - mapping = self.get_mapping(self.current_mapping) - percent_complete = self.percent_complete + (100.0 / self.total_pages) - fields = dict(percent_complete=percent_complete) - if self.current_mapping_action == "Insert": - start = self.current_mapping_start + mapping.page_length - fields["current_mapping_start"] = start - elif self.current_mapping_action == "Delete": - delete_start = self.current_mapping_delete_start + mapping.page_length - fields["current_mapping_delete_start"] = delete_start - - self.db_set(fields, notify=True, commit=True) - - if percent_complete < 100: - frappe.publish_realtime( - self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user - ) - - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - - def run_current_mapping(self): - try: - mapping = self.get_mapping(self.current_mapping) - - if mapping.mapping_type == "Push": - done = self.push() - elif mapping.mapping_type == "Pull": - done = self.pull() - - if done: - self.enqueue_next_mapping() - else: - self.enqueue_next_page() - - except Exception as e: - self.db_set("status", "Error", notify=True, commit=True) - print("Data Migration Run failed") - print(frappe.get_traceback()) - self.execute_postprocess("Error") - raise e - - def get_last_modified_condition(self): - last_run_timestamp = frappe.db.get_value( - "Data Migration Run", - dict( - data_migration_plan=self.data_migration_plan, - data_migration_connector=self.data_migration_connector, - name=("!=", self.name), - ), - "modified", - ) - if last_run_timestamp: - condition = dict(modified=(">", last_run_timestamp)) - else: - condition = {} - return condition - - def begin(self): - plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled] - self.mappings = [ - frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings - ] - - total_pages = 0 - for m in [mapping for mapping in self.mappings]: - if m.mapping_type == "Push": - count = float(self.get_count(m)) - page_count = math.ceil(count / m.page_length) - total_pages += page_count - if m.mapping_type == "Pull": - total_pages += 10 - - self.db_set( - dict( - status="Started", - current_mapping=None, - current_mapping_start=0, - current_mapping_delete_start=0, - percent_complete=0, - current_mapping_action="Insert", - total_pages=total_pages, - ), - notify=True, - commit=True, - ) - - def complete(self): - fields = dict() - - push_failed = self.get_log("push_failed", []) - pull_failed = self.get_log("pull_failed", []) - - status = "Partial Success" - - if not push_failed and not pull_failed: - status = "Success" - fields["percent_complete"] = 100 - - fields["status"] = status - - self.db_set(fields, notify=True, commit=True) - - self.execute_postprocess(status) - - frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user) - - def execute_postprocess(self, status): - # Execute post process - postprocess_method_path = self.get_plan().postprocess_method - - if postprocess_method_path: - frappe.get_attr(postprocess_method_path)( - { - "status": status, - "stats": { - "push_insert": self.push_insert, - "push_update": self.push_update, - "push_delete": self.push_delete, - "pull_insert": self.pull_insert, - "pull_update": self.pull_update, - }, - } - ) - - def get_plan(self): - if not hasattr(self, "plan"): - self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan) - return self.plan - - def get_mapping(self, mapping_name): - if hasattr(self, "mappings"): - for m in self.mappings: - if m.name == mapping_name: - return m - return frappe.get_doc("Data Migration Mapping", mapping_name) - - def get_next_mapping_name(self): - mappings = [m for m in self.get_plan().mappings if m.enabled] - if not self.current_mapping: - # first - return mappings[0].mapping - for i, d in enumerate(mappings): - if i == len(mappings) - 1: - # last - return None - if d.mapping == self.current_mapping: - return mappings[i + 1].mapping - - raise frappe.ValidationError("Mapping Broken") - - def get_data(self, filters): - mapping = self.get_mapping(self.current_mapping) - or_filters = self.get_or_filters(mapping) - start = self.current_mapping_start - - data = [] - doclist = frappe.get_all( - mapping.local_doctype, - filters=filters, - or_filters=or_filters, - start=start, - page_length=mapping.page_length, - ) - - for d in doclist: - doc = frappe.get_doc(mapping.local_doctype, d["name"]) - data.append(doc) - return data - - def get_new_local_data(self): - """Fetch newly inserted local data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # new docs dont have migration field set - filters.update({mapping.migration_id_field: ""}) - - return self.get_data(filters) - - def get_updated_local_data(self): - """Fetch local updated data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # existing docs must have migration field set - filters.update({mapping.migration_id_field: ("!=", "")}) - - return self.get_data(filters) - - def get_deleted_local_data(self): - """Fetch local deleted data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = self.get_last_modified_condition() - filters.update({"deleted_doctype": mapping.local_doctype}) - - data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters) - - _data = [] - for d in data: - doc = json.loads(d.data) - if doc.get(mapping.migration_id_field): - doc["_deleted_document_name"] = d["name"] - _data.append(doc) - - return _data - - def get_remote_data(self): - """Fetch data from remote using `connection.get`. Used during Pull""" - mapping = self.get_mapping(self.current_mapping) - start = self.current_mapping_start - filters = mapping.get_filters() or {} - connection = self.get_connection() - - return connection.get( - mapping.remote_objectname, - fields=["*"], - filters=filters, - start=start, - page_length=mapping.page_length, - ) - - def get_count(self, mapping): - filters = mapping.get_filters() or {} - or_filters = self.get_or_filters(mapping) - - to_insert = frappe.get_all( - mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters - )[0].total - - to_delete = frappe.get_all( - "Deleted Document", - ["count(name) as total"], - filters={"deleted_doctype": mapping.local_doctype}, - or_filters=or_filters, - )[0].total - - return to_insert + to_delete - - def get_or_filters(self, mapping): - or_filters = self.get_last_modified_condition() - - # docs whose migration_id_field is not set - # failed in the previous run, include those too - or_filters.update({mapping.migration_id_field: ("=", "")}) - - return or_filters - - def get_connection(self): - if not hasattr(self, "connection"): - self.connection = frappe.get_doc( - "Data Migration Connector", self.data_migration_connector - ).get_connection() - - return self.connection - - def push(self): - self.db_set("current_mapping_type", "Push") - done = True - - if self.current_mapping_action == "Insert": - done = self._push_insert() - - elif self.current_mapping_action == "Update": - done = self._push_update() - - elif self.current_mapping_action == "Delete": - done = self._push_delete() - - return done - - def _push_insert(self): - """Inserts new local docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_new_local_data() - - for d in data: - # pre process before insert - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - try: - response_doc = connection.insert(mapping.remote_objectname, doc) - frappe.db.set_value( - mapping.local_doctype, - d.name, - mapping.migration_id_field, - response_doc[connection.name_field], - update_modified=False, - ) - frappe.db.commit() - self.update_log("push_insert", 1) - # post process after insert - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to insert - self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_update(self): - """Updates local modified docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_updated_local_data() - - for d in data: - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - try: - response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value) - self.update_log("push_update", 1) - # post process after update - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more data to update - self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_delete(self): - """Deletes docs deleted from local on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_deleted_local_data() - - for d in data: - # Deleted Document also has a custom field for migration_id - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - self.pre_process_doc(d) - try: - response_doc = connection.delete(mapping.remote_objectname, migration_id_value) - self.update_log("push_delete", 1) - # post process only when action is success - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to delete - # done with this mapping - return True - - def pull(self): - self.db_set("current_mapping_type", "Pull") - - connection = self.get_connection() - mapping = self.get_mapping(self.current_mapping) - data = self.get_remote_data() - - for d in data: - migration_id_value = get_source_value(d, connection.name_field) - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - if migration_id_value: - try: - if not local_doc_exists(mapping, migration_id_value): - # insert new local doc - local_doc = insert_local_doc(mapping, doc) - - self.update_log("pull_insert", 1) - # set migration id - frappe.db.set_value( - mapping.local_doctype, - local_doc.name, - mapping.migration_id_field, - migration_id_value, - update_modified=False, - ) - frappe.db.commit() - else: - # update doc - local_doc = update_local_doc(mapping, doc, migration_id_value) - self.update_log("pull_update", 1) - # post process doc after success - self.post_process_doc(remote_doc=d, local_doc=local_doc) - except Exception as e: - # failed, append to log - self.update_log("pull_failed", {migration_id_value: cstr(e)}) - - if len(data) < mapping.page_length: - # last page, done with pull - return True - - def pre_process_doc(self, doc): - plan = self.get_plan() - doc = plan.pre_process_doc(self.current_mapping, doc) - return doc - - def post_process_doc(self, local_doc=None, remote_doc=None): - plan = self.get_plan() - doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc) - return doc - - def set_log(self, key, value): - value = json.dumps(value) if "_failed" in key else value - self.db_set(key, value) - - def update_log(self, key, value=None): - """ - Helper for updating logs, - push_failed and pull_failed are stored as json, - other keys are stored as int - """ - if "_failed" in key: - # json - self.set_log(key, self.get_log(key, []) + [value]) - else: - # int - self.set_log(key, self.get_log(key, 0) + (value or 1)) - - def get_log(self, key, default=None): - value = self.db_get(key) - if "_failed" in key: - if not value: - value = json.dumps(default) - value = json.loads(value) - return value or default - - -def insert_local_doc(mapping, doc): - try: - # insert new doc - if not doc.doctype: - doc.doctype = mapping.local_doctype - doc = frappe.get_doc(doc).insert() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull insert") - print(frappe.get_traceback()) - return None - - -def update_local_doc(mapping, remote_doc, migration_id_value): - try: - # migration id value is set in migration_id_field in mapping.local_doctype - docname = frappe.db.get_value( - mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value} - ) - - doc = frappe.get_doc(mapping.local_doctype, docname) - doc.update(remote_doc) - doc.save() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull update") - print(frappe.get_traceback()) - return None - - -def local_doc_exists(mapping, migration_id_value): - return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value}) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py deleted file mode 100644 index 0357b1e0f5..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - -import frappe - - -class TestDataMigrationRun(unittest.TestCase): - def test_run(self): - create_plan() - - description = "data migration todo" - new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert() - - event_subject = "data migration event" - frappe.get_doc( - dict( - doctype="Event", - subject=event_subject, - repeat_on="Monthly", - starts_on=frappe.utils.now_datetime(), - ) - ).insert() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - self.assertEqual(run.db_get("status"), "Success") - - self.assertEqual(run.db_get("push_insert"), 1) - self.assertEqual(run.db_get("pull_insert"), 1) - - todo = frappe.get_doc("ToDo", new_todo.name) - self.assertTrue(todo.todo_sync_id) - - # Pushed Event - event = frappe.get_doc("Event", todo.todo_sync_id) - self.assertEqual(event.subject, description) - - # Pulled ToDo - created_todo = frappe.get_doc("ToDo", {"description": event_subject}) - self.assertEqual(created_todo.description, event_subject) - - todo_list = frappe.get_list( - "ToDo", filters={"description": "data migration todo"}, fields=["name"] - ) - todo_name = todo_list[0].name - - todo = frappe.get_doc("ToDo", todo_name) - todo.description = "data migration todo updated" - todo.save() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - - # Update - self.assertEqual(run.db_get("status"), "Success") - self.assertEqual(run.db_get("pull_update"), 1) - - -def create_plan(): - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Todo to Event", - "remote_objectname": "Event", - "remote_primary_key": "name", - "mapping_type": "Push", - "local_doctype": "ToDo", - "fields": [ - {"remote_fieldname": "subject", "local_fieldname": "description"}, - { - "remote_fieldname": "starts_on", - "local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())", - }, - ], - "condition": '{"description": "data migration todo" }', - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Event to ToDo", - "remote_objectname": "Event", - "remote_primary_key": "name", - "local_doctype": "ToDo", - "local_primary_key": "name", - "mapping_type": "Pull", - "condition": '{"subject": "data migration event" }', - "fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Plan", - "plan_name": "ToDo Sync", - "module": "Core", - "mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Connector", - "connector_name": "Local Connector", - "connector_type": "Frappe", - # connect to same host. - "hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), - "username": "Administrator", - "password": frappe.conf.get("admin_password") or "admin", - } - ).insert(ignore_if_duplicate=True) diff --git a/frappe/desk/doctype/route_history/route_history.json b/frappe/desk/doctype/route_history/route_history.json index 09db2320ca..a5d73fc360 100644 --- a/frappe/desk/doctype/route_history/route_history.json +++ b/frappe/desk/doctype/route_history/route_history.json @@ -1,126 +1,52 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-10-05 11:26:04.601113", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "route", + "user" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "route", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, - "label": "Route", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Route" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "user", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, + "in_standard_filter": 1, "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "User" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 13:26:03.106050", + "links": [], + "modified": "2022-06-13 05:48:56.967244", "modified_by": "Administrator", "module": "Desk", "name": "Route History", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_seen": 0, - "track_views": 0 -} + "states": [], + "title_field": "route" +} \ No newline at end of file diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 4c535b2811..93b883cda6 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -53,22 +53,6 @@ def sync_for(app_name, force=0, reset_permissions=False): os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json") ) - for data_migration_module in [ - "data_migration_mapping_detail", - "data_migration_mapping", - "data_migration_plan_mapping", - "data_migration_plan", - ]: - files.append( - os.path.join( - FRAPPE_PATH, - "data_migration", - "doctype", - data_migration_module, - f"{data_migration_module}.json", - ) - ) - for desk_module in [ "number_card", "dashboard_chart", @@ -124,8 +108,6 @@ def get_doc_files(files, start_path): "web_template", "notification", "print_style", - "data_migration_mapping", - "data_migration_plan", "workspace", "onboarding_step", "module_onboarding", diff --git a/frappe/modules.txt b/frappe/modules.txt index a707ca853e..fb7817f6ba 100644 --- a/frappe/modules.txt +++ b/frappe/modules.txt @@ -8,7 +8,6 @@ Desk Integrations Printing Contacts -Data Migration Social Automation Event Streaming \ No newline at end of file diff --git a/frappe/patches.txt b/frappe/patches.txt index 6c46d5dcd9..b771485c3c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -201,3 +201,4 @@ frappe.patches.v14_0.update_color_names_in_kanban_board_column frappe.patches.v14_0.update_is_system_generated_flag frappe.patches.v14_0.update_auto_account_deletion_duration frappe.patches.v14_0.set_document_expiry_default +frappe.patches.v14_0.delete_data_migration_tool \ No newline at end of file diff --git a/frappe/patches/v14_0/delete_data_migration_tool.py b/frappe/patches/v14_0/delete_data_migration_tool.py new file mode 100644 index 0000000000..d0416cb1e7 --- /dev/null +++ b/frappe/patches/v14_0/delete_data_migration_tool.py @@ -0,0 +1,12 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe + + +def execute(): + doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name") + for doctype in doctypes: + frappe.delete_doc("DocType", doctype, ignore_missing=True) + + frappe.delete_doc("Module Def", "Data Migration", ignore_missing=True, force=True) diff --git a/frappe/public/js/frappe/form/quick_entry.js b/frappe/public/js/frappe/form/quick_entry.js index 86523d7088..f55f7139c9 100644 --- a/frappe/public/js/frappe/form/quick_entry.js +++ b/frappe/public/js/frappe/form/quick_entry.js @@ -267,7 +267,7 @@ frappe.ui.form.QuickEntryForm = class QuickEntryForm { render_edit_in_full_page_link() { var me = this; this.dialog.add_custom_action( - `${__("Edit in full page")}`, + `${__("Edit Full Form")}`, () => me.open_doc(true) ); }