vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor!: Remove Data Migration Tool (#17134)

Browse source 
* refactor: Remove Data Migration Tool

* chore: unnecessary imports
This commit is contained in:
Faris Ansari 2022-06-13 16:24:00 +05:30 • committed by GitHub
parent 63a5db94cf
commit eea5cc52da
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 13 additions and 3197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
frappe/data_migration/__init__.py
View file

0
frappe/data_migration/doctype/__init__.py
View file

0
frappe/data_migration/doctype/data_migration_connector/__init__.py
View file

0
frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py
View file

24
frappe/data_migration/doctype/data_migration_connector/connectors/base.py
View file

@ -1,24 +0,0 @@
from abc import ABCMeta, abstractmethod
from frappe.utils.password import get_decrypted_password
class BaseConnection(metaclass=ABCMeta):
@abstractmethod
def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10):
pass
@abstractmethod
def insert(self, doctype, doc):
pass
@abstractmethod
def update(self, doctype, doc, migration_id):
pass
@abstractmethod
def delete(self, doctype, migration_id):
pass
def get_password(self):
return get_decrypted_password("Data Migration Connector", self.connector.name)

32
frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py
View file

@ -1,32 +0,0 @@
import frappe
from frappe.frappeclient import FrappeClient
from .base import BaseConnection
class FrappeConnection(BaseConnection):
def __init__(self, connector):
self.connector = connector
self.connection = FrappeClient(
self.connector.hostname, self.connector.username, self.get_password()
)
self.name_field = "name"
def insert(self, doctype, doc):
doc = frappe._dict(doc)
doc.doctype = doctype
return self.connection.insert(doc)
def update(self, doctype, doc, migration_id):
doc = frappe._dict(doc)
doc.doctype = doctype
doc.name = migration_id
return self.connection.update(doc)
def delete(self, doctype, migration_id):
return self.connection.delete(doctype, migration_id)
def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20):
return self.connection.get_list(
doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length
)

47
frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js
View file

@ -1,47 +0,0 @@
// Copyright (c) 2017, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Migration Connector', {
onload(frm) {
if(frappe.boot.developer_mode) {
frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm));
}
},
new_connection(frm) {
const d = new frappe.ui.Dialog({
title: __('New Connection'),
fields: [
{ label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 },
{ label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 },
],
primary_action_label: __('Create'),
primary_action: (values) => {
let { module, connection_name } = values;
frm.events.create_new_connection(module, connection_name)
.then(r => {
if (r.message) {
const connector_name = connection_name
.replace('connection', 'Connector')
.replace('Connection', 'Connector')
.trim();
frm.set_value('connector_name', connector_name);
frm.set_value('connector_type', 'Custom');
frm.set_value('python_module', r.message);
frm.save();
frappe.show_alert(__("New module created {0}", [r.message]));
d.hide();
}
});
}
});
d.show();
},
create_new_connection(module, connection_name) {
return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', {
module, connection_name
});
}
});

307
frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json
View file

@ -1,307 +0,0 @@
{
"allow_copy": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 1,
"autoname": "field:connector_name",
"beta": 1,
"creation": "2017-08-11 05:03:27.091416",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "connector_name",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Connector Name",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:!doc.is_custom",
"fieldname": "connector_type",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Connector Type",
"length": 0,
"no_copy": 0,
"options": "\nFrappe\nCustom",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:doc.connector_type == 'Custom'",
"fieldname": "python_module",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Python Module",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "authentication_credentials",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Authentication Credentials",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "",
"fieldname": "hostname",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Hostname",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "database_name",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Database Name",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "username",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Username",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "password",
"fieldtype": "Password",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Password",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2017-12-01 13:38:55.992499",
"modified_by": "Administrator",
"module": "Data Migration",
"name": "Data Migration Connector",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"apply_user_permissions": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"quick_entry": 0,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0
}

107
frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py
View file

@ -1,107 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# License: MIT. See LICENSE
import os
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.modules.export_file import create_init_py
from .connectors.base import BaseConnection
from .connectors.frappe_connection import FrappeConnection
class DataMigrationConnector(Document):
def validate(self):
if not (self.python_module or self.connector_type):
frappe.throw(_("Enter python module or select connector type"))
if self.python_module:
try:
get_connection_class(self.python_module)
except:
frappe.throw(frappe._("Invalid module path"))
def get_connection(self):
if self.python_module:
_class = get_connection_class(self.python_module)
return _class(self)
else:
self.connection = FrappeConnection(self)
return self.connection
@frappe.whitelist()
def create_new_connection(module, connection_name):
if not frappe.conf.get("developer_mode"):
frappe.msgprint(_("Please enable developer mode to create new connection"))
return
# create folder
module_path = frappe.get_module_path(module)
connectors_folder = os.path.join(module_path, "connectors")
frappe.create_folder(connectors_folder)
# create init py
create_init_py(module_path, "connectors", "")
connection_class = connection_name.replace(" ", "")
file_name = frappe.scrub(connection_name) + ".py"
file_path = os.path.join(module_path, "connectors", file_name)
# create boilerplate file
with open(file_path, "w") as f:
f.write(connection_boilerplate.format(connection_class=connection_class))
# get python module string from file_path
app_name = frappe.db.get_value("Module Def", module, "app_name")
python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace(
os.path.sep, "."
)[:-3]
return python_module
def get_connection_class(python_module):
filename = python_module.rsplit(".", 1)[-1]
classname = frappe.unscrub(filename).replace(" ", "")
module = frappe.get_module(python_module)
raise_error = False
if hasattr(module, classname):
_class = getattr(module, classname)
if not issubclass(_class, BaseConnection):
raise_error = True
else:
raise_error = True
if raise_error:
raise ImportError(filename)
return _class
connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection
class {connection_class}(BaseConnection):
def __init__(self, connector):
# self.connector = connector
# self.connection = YourModule(self.connector.username, self.get_password())
# self.name_field = 'id'
pass
def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10):
pass
def insert(self, doctype, doc):
pass
def update(self, doctype, doc, migration_id):
pass
def delete(self, doctype, migration_id):
pass
"""

8
frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py
View file

@ -1,8 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# License: MIT. See LICENSE
import unittest
class TestDataMigrationConnector(unittest.TestCase):
pass

0
frappe/data_migration/doctype/data_migration_mapping/__init__.py
View file

8
frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js
View file

@ -1,8 +0,0 @@
// Copyright (c) 2017, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Migration Mapping', {
refresh: function() {
}
});

456
frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json
View file

@ -1,456 +0,0 @@
{
"allow_copy": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 1,
"autoname": "field:mapping_name",
"beta": 1,
"creation": "2017-08-11 05:11:49.975801",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "mapping_name",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Mapping Name",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "remote_objectname",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Remote Objectname",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "remote_primary_key",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Remote Primary Key",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "local_doctype",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Local DocType",
"length": 0,
"no_copy": 0,
"options": "DocType",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "local_primary_key",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Local Primary Key",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "column_break_5",
"fieldtype": "Column Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "mapping_type",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Mapping Type",
"length": 0,
"no_copy": 0,
"options": "Push\nPull\nSync",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "10",
"fieldname": "page_length",
"fieldtype": "Int",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Page Length",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "migration_id_field",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Migration ID Field",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "mapping",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Mapping",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "fields",
"fieldtype": "Table",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Field Maps",
"length": 0,
"no_copy": 0,
"options": "Data Migration Mapping Detail",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 1,
"columns": 0,
"fieldname": "condition_detail",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Condition Detail",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "condition",
"fieldtype": "Code",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Condition",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2017-09-27 18:06:43.275207",
"modified_by": "Administrator",
"module": "Data Migration",
"name": "Data Migration Mapping",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"apply_user_permissions": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"quick_entry": 1,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0
}

83
frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py
View file

@ -1,83 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# License: MIT. See LICENSE
import frappe
from frappe.model.document import Document
from frappe.utils.safe_exec import get_safe_globals
class DataMigrationMapping(Document):
def get_filters(self):
if self.condition:
return frappe.safe_eval(self.condition, get_safe_globals())
def get_fields(self):
fields = []
for f in self.fields:
if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")):
fields.append(f.local_fieldname)
if frappe.db.has_column(self.local_doctype, self.migration_id_field):
fields.append(self.migration_id_field)
if "name" not in fields:
fields.append("name")
return fields
def get_mapped_record(self, doc):
"""Build a mapped record using information from the fields table"""
mapped = frappe._dict()
key_fieldname = "remote_fieldname"
value_fieldname = "local_fieldname"
if self.mapping_type == "Pull":
key_fieldname, value_fieldname = value_fieldname, key_fieldname
for field_map in self.fields:
key = get_source_value(field_map, key_fieldname)
if not field_map.is_child_table:
# field to field mapping
value = get_value_from_fieldname(field_map, value_fieldname, doc)
else:
# child table mapping
mapping_name = field_map.child_table_mapping
value = get_mapped_child_records(
mapping_name, doc.get(get_source_value(field_map, value_fieldname))
)
mapped[key] = value
return mapped
def get_mapped_child_records(mapping_name, child_docs):
mapped_child_docs = []
mapping = frappe.get_doc("Data Migration Mapping", mapping_name)
for child_doc in child_docs:
mapped_child_docs.append(mapping.get_mapped_record(child_doc))
return mapped_child_docs
def get_value_from_fieldname(field_map, fieldname_field, doc):
field_name = get_source_value(field_map, fieldname_field)
if field_name.startswith("eval:"):
value = frappe.safe_eval(field_name[5:], get_safe_globals())
elif field_name[0] in ('"', "'"):
value = field_name[1:-1]
else:
value = get_source_value(doc, field_name)
return value
def get_source_value(source, key):
"""Get value from source (object or dict) based on key"""
if isinstance(source, dict):
return source.get(key)
else:
return getattr(source, key)

8
frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py
View file

@ -1,8 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# License: MIT. See LICENSE
import unittest
class TestDataMigrationMapping(unittest.TestCase):
pass

0
frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py
View file

163
frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json
View file

@ -1,163 +0,0 @@
{
"allow_copy": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"beta": 0,
"creation": "2017-08-11 05:09:10.900237",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "remote_fieldname",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Remote Fieldname",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "local_fieldname",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Local Fieldname",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "is_child_table",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Is Child Table",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "is_child_table",
"fieldname": "child_table_mapping",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Child Table Mapping",
"length": 0,
"no_copy": 0,
"options": "Data Migration Mapping",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 1,
"max_attachments": 0,
"modified": "2017-09-28 17:13:31.337005",
"modified_by": "Administrator",
"module": "Data Migration",
"name": "Data Migration Mapping Detail",
"name_case": "",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0
}

9
frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py
View file

@ -1,9 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# License: MIT. See LICENSE
from frappe.model.document import Document
class DataMigrationMappingDetail(Document):
pass

0
frappe/data_migration/doctype/data_migration_plan/__init__.py
View file

10
frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js
View file

@ -1,10 +0,0 @@
// Copyright (c) 2017, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Migration Plan', {
onload(frm) {
frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', {
data_migration_plan: frm.doc.name
}));
}
});

224
frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json
View file

@ -1,224 +0,0 @@
{
"allow_copy": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"autoname": "field:plan_name",
"beta": 0,
"creation": "2017-08-11 05:15:51.482165",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "plan_name",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Plan Name",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 1
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "module",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Module",
"length": 0,
"no_copy": 0,
"options": "Module Def",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "mappings",
"fieldtype": "Table",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Mappings",
"length": 0,
"no_copy": 0,
"options": "Data Migration Plan Mapping",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "preprocess_method",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Preprocess Method",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "postprocess_method",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Postprocess Method",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2020-09-18 17:26:09.703215",
"modified_by": "Administrator",
"module": "Data Migration",
"name": "Data Migration Plan",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"quick_entry": 1,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0
}

78
frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py
View file

@ -1,78 +0,0 @@
# Copyright (c) 2021, Frappe Technologies and contributors
# License: MIT. See LICENSE
import frappe
from frappe.custom.doctype.custom_field.custom_field import create_custom_field
from frappe.model.document import Document
from frappe.modules import get_module_path, scrub_dt_dn
from frappe.modules.export_file import create_init_py, export_to_files
def get_mapping_module(module, mapping_name):
app_name = frappe.db.get_value("Module Def", module, "app_name")
mapping_name = frappe.scrub(mapping_name)
module = frappe.scrub(module)
try:
return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}")
except ImportError:
return None
class DataMigrationPlan(Document):
def on_update(self):
# update custom fields in mappings
self.make_custom_fields_for_mappings()
if frappe.flags.in_import or frappe.flags.in_test:
return
if frappe.local.conf.get("developer_mode"):
record_list = [["Data Migration Plan", self.name]]
for m in self.mappings:
record_list.append(["Data Migration Mapping", m.mapping])
export_to_files(record_list=record_list, record_module=self.module)
for m in self.mappings:
dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping)
create_init_py(get_module_path(self.module), dt, dn)
def make_custom_fields_for_mappings(self):
frappe.flags.ignore_in_install = True
label = self.name + " ID"
fieldname = frappe.scrub(label)
df = {
"label": label,
"fieldname": fieldname,
"fieldtype": "Data",
"hidden": 1,
"read_only": 1,
"unique": 1,
"no_copy": 1,
}
for m in self.mappings:
mapping = frappe.get_doc("Data Migration Mapping", m.mapping)
create_custom_field(mapping.local_doctype, df)
mapping.migration_id_field = fieldname
mapping.save()
# Create custom field in Deleted Document
create_custom_field("Deleted Document", df)
frappe.flags.ignore_in_install = False
def pre_process_doc(self, mapping_name, doc):
module = get_mapping_module(self.module, mapping_name)
if module and hasattr(module, "pre_process"):
return module.pre_process(doc)
return doc
def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None):
module = get_mapping_module(self.module, mapping_name)
if module and hasattr(module, "post_process"):
return module.post_process(local_doc=local_doc, remote_doc=remote_doc)

8
frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py
View file

@ -1,8 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# License: MIT. See LICENSE
import unittest
class TestDataMigrationPlan(unittest.TestCase):
pass

0
frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py
View file

103
frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json
View file

@ -1,103 +0,0 @@
{
"allow_copy": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"beta": 1,
"creation": "2017-08-11 05:15:38.390831",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "mapping",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Mapping",
"length": 0,
"no_copy": 0,
"options": "Data Migration Mapping",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "1",
"fieldname": "enabled",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Enabled",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 1,
"max_attachments": 0,
"modified": "2017-09-20 21:43:04.908650",
"modified_by": "Administrator",
"module": "Data Migration",
"name": "Data Migration Plan Mapping",
"name_case": "",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0
}

9
frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py
View file

@ -1,9 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# License: MIT. See LICENSE
from frappe.model.document import Document
class DataMigrationPlanMapping(Document):
pass

0
frappe/data_migration/doctype/data_migration_run/__init__.py
View file

14
frappe/data_migration/doctype/data_migration_run/data_migration_run.js
View file

@ -1,14 +0,0 @@
// Copyright (c) 2017, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Migration Run', {
refresh: function(frm) {
if (frm.doc.status !== 'Success') {
frm.add_custom_button(__('Run'), () => frm.call('run'));
}
if (frm.doc.status === 'Started') {
frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete,
__('Currently updating {0}', [frm.doc.current_mapping]));
}
}
});

838
frappe/data_migration/doctype/data_migration_run/data_migration_run.json
View file

@ -1,838 +0,0 @@
{
"allow_copy": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"beta": 0,
"creation": "2017-09-11 12:55:27.597728",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "data_migration_plan",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Data Migration Plan",
"length": 0,
"no_copy": 0,
"options": "Data Migration Plan",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "data_migration_connector",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Data Migration Connector",
"length": 0,
"no_copy": 0,
"options": "Data Migration Connector",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "Pending",
"fieldname": "status",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Status",
"length": 0,
"no_copy": 1,
"options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "start_time",
"fieldtype": "Datetime",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Start Time",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "end_time",
"fieldtype": "Datetime",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "End Time",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "remote_id",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Remote ID",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "current_mapping",
"fieldtype": "Data",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Current Mapping",
"length": 0,
"no_copy": 1,
"options": "",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "current_mapping_start",
"fieldtype": "Int",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Current Mapping Start",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "current_mapping_delete_start",
"fieldtype": "Int",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Current Mapping Delete Start",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "current_mapping_type",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Current Mapping Type",
"length": 0,
"no_copy": 0,
"options": "Push\nPull",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:(doc.status !== 'Pending')",
"fieldname": "current_mapping_action",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Current Mapping Action",
"length": 0,
"no_copy": 1,
"options": "Insert\nDelete",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "total_pages",
"fieldtype": "Int",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Total Pages",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "percent_complete",
"fieldtype": "Percent",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Percent Complete",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "trigger_name",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Trigger Name",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:(doc.status !== 'Pending')",
"fieldname": "logs_sb",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Logs",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "push_insert",
"fieldtype": "Int",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Push Insert",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "push_update",
"fieldtype": "Int",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Push Update",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "push_delete",
"fieldtype": "Int",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Push Delete",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "push_failed",
"fieldtype": "Code",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Push Failed",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "column_break_16",
"fieldtype": "Column Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "pull_insert",
"fieldtype": "Int",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Pull Insert",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "pull_update",
"fieldtype": "Int",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Pull Update",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "pull_failed",
"fieldtype": "Code",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Pull Failed",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:doc.failed_log !== '[]'",
"fieldname": "log",
"fieldtype": "Code",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Log",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2020-09-18 17:26:09.703215",
"modified_by": "Administrator",
"module": "Data Migration",
"name": "Data Migration Run",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"quick_entry": 0,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0
}

514
frappe/data_migration/doctype/data_migration_run/data_migration_run.py
View file

@ -1,514 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# License: MIT. See LICENSE
import json
import math
import frappe
from frappe import _
from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import (
get_source_value,
)
from frappe.model.document import Document
from frappe.utils import cstr
class DataMigrationRun(Document):
@frappe.whitelist()
def run(self):
self.begin()
if self.total_pages > 0:
self.enqueue_next_mapping()
else:
self.complete()
def enqueue_next_mapping(self):
next_mapping_name = self.get_next_mapping_name()
if next_mapping_name:
next_mapping = self.get_mapping(next_mapping_name)
self.db_set(
dict(
current_mapping=next_mapping.name,
current_mapping_start=0,
current_mapping_delete_start=0,
current_mapping_action="Insert",
),
notify=True,
commit=True,
)
frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test)
else:
self.complete()
def enqueue_next_page(self):
mapping = self.get_mapping(self.current_mapping)
percent_complete = self.percent_complete + (100.0 / self.total_pages)
fields = dict(percent_complete=percent_complete)
if self.current_mapping_action == "Insert":
start = self.current_mapping_start + mapping.page_length
fields["current_mapping_start"] = start
elif self.current_mapping_action == "Delete":
delete_start = self.current_mapping_delete_start + mapping.page_length
fields["current_mapping_delete_start"] = delete_start
self.db_set(fields, notify=True, commit=True)
if percent_complete < 100:
frappe.publish_realtime(
self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user
)
frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test)
def run_current_mapping(self):
try:
mapping = self.get_mapping(self.current_mapping)
if mapping.mapping_type == "Push":
done = self.push()
elif mapping.mapping_type == "Pull":
done = self.pull()
if done:
self.enqueue_next_mapping()
else:
self.enqueue_next_page()
except Exception as e:
self.db_set("status", "Error", notify=True, commit=True)
print("Data Migration Run failed")
print(frappe.get_traceback())
self.execute_postprocess("Error")
raise e
def get_last_modified_condition(self):
last_run_timestamp = frappe.db.get_value(
"Data Migration Run",
dict(
data_migration_plan=self.data_migration_plan,
data_migration_connector=self.data_migration_connector,
name=("!=", self.name),
),
"modified",
)
if last_run_timestamp:
condition = dict(modified=(">", last_run_timestamp))
else:
condition = {}
return condition
def begin(self):
plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled]
self.mappings = [
frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings
]
total_pages = 0
for m in [mapping for mapping in self.mappings]:
if m.mapping_type == "Push":
count = float(self.get_count(m))
page_count = math.ceil(count / m.page_length)
total_pages += page_count
if m.mapping_type == "Pull":
total_pages += 10
self.db_set(
dict(
status="Started",
current_mapping=None,
current_mapping_start=0,
current_mapping_delete_start=0,
percent_complete=0,
current_mapping_action="Insert",
total_pages=total_pages,
),
notify=True,
commit=True,
)
def complete(self):
fields = dict()
push_failed = self.get_log("push_failed", [])
pull_failed = self.get_log("pull_failed", [])
status = "Partial Success"
if not push_failed and not pull_failed:
status = "Success"
fields["percent_complete"] = 100
fields["status"] = status
self.db_set(fields, notify=True, commit=True)
self.execute_postprocess(status)
frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user)
def execute_postprocess(self, status):
# Execute post process
postprocess_method_path = self.get_plan().postprocess_method
if postprocess_method_path:
frappe.get_attr(postprocess_method_path)(
{
"status": status,
"stats": {
"push_insert": self.push_insert,
"push_update": self.push_update,
"push_delete": self.push_delete,
"pull_insert": self.pull_insert,
"pull_update": self.pull_update,
},
}
)
def get_plan(self):
if not hasattr(self, "plan"):
self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan)
return self.plan
def get_mapping(self, mapping_name):
if hasattr(self, "mappings"):
for m in self.mappings:
if m.name == mapping_name:
return m
return frappe.get_doc("Data Migration Mapping", mapping_name)
def get_next_mapping_name(self):
mappings = [m for m in self.get_plan().mappings if m.enabled]
if not self.current_mapping:
# first
return mappings[0].mapping
for i, d in enumerate(mappings):
if i == len(mappings) - 1:
# last
return None
if d.mapping == self.current_mapping:
return mappings[i + 1].mapping
raise frappe.ValidationError("Mapping Broken")
def get_data(self, filters):
mapping = self.get_mapping(self.current_mapping)
or_filters = self.get_or_filters(mapping)
start = self.current_mapping_start
data = []
doclist = frappe.get_all(
mapping.local_doctype,
filters=filters,
or_filters=or_filters,
start=start,
page_length=mapping.page_length,
)
for d in doclist:
doc = frappe.get_doc(mapping.local_doctype, d["name"])
data.append(doc)
return data
def get_new_local_data(self):
"""Fetch newly inserted local data using `frappe.get_all`. Used during Push"""
mapping = self.get_mapping(self.current_mapping)
filters = mapping.get_filters() or {}
# new docs dont have migration field set
filters.update({mapping.migration_id_field: ""})
return self.get_data(filters)
def get_updated_local_data(self):
"""Fetch local updated data using `frappe.get_all`. Used during Push"""
mapping = self.get_mapping(self.current_mapping)
filters = mapping.get_filters() or {}
# existing docs must have migration field set
filters.update({mapping.migration_id_field: ("!=", "")})
return self.get_data(filters)
def get_deleted_local_data(self):
"""Fetch local deleted data using `frappe.get_all`. Used during Push"""
mapping = self.get_mapping(self.current_mapping)
filters = self.get_last_modified_condition()
filters.update({"deleted_doctype": mapping.local_doctype})
data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters)
_data = []
for d in data:
doc = json.loads(d.data)
if doc.get(mapping.migration_id_field):
doc["_deleted_document_name"] = d["name"]
_data.append(doc)
return _data
def get_remote_data(self):
"""Fetch data from remote using `connection.get`. Used during Pull"""
mapping = self.get_mapping(self.current_mapping)
start = self.current_mapping_start
filters = mapping.get_filters() or {}
connection = self.get_connection()
return connection.get(
mapping.remote_objectname,
fields=["*"],
filters=filters,
start=start,
page_length=mapping.page_length,
)
def get_count(self, mapping):
filters = mapping.get_filters() or {}
or_filters = self.get_or_filters(mapping)
to_insert = frappe.get_all(
mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters
)[0].total
to_delete = frappe.get_all(
"Deleted Document",
["count(name) as total"],
filters={"deleted_doctype": mapping.local_doctype},
or_filters=or_filters,
)[0].total
return to_insert + to_delete
def get_or_filters(self, mapping):
or_filters = self.get_last_modified_condition()
# docs whose migration_id_field is not set
# failed in the previous run, include those too
or_filters.update({mapping.migration_id_field: ("=", "")})
return or_filters
def get_connection(self):
if not hasattr(self, "connection"):
self.connection = frappe.get_doc(
"Data Migration Connector", self.data_migration_connector
).get_connection()
return self.connection
def push(self):
self.db_set("current_mapping_type", "Push")
done = True
if self.current_mapping_action == "Insert":
done = self._push_insert()
elif self.current_mapping_action == "Update":
done = self._push_update()
elif self.current_mapping_action == "Delete":
done = self._push_delete()
return done
def _push_insert(self):
"""Inserts new local docs on remote"""
mapping = self.get_mapping(self.current_mapping)
connection = self.get_connection()
data = self.get_new_local_data()
for d in data:
# pre process before insert
doc = self.pre_process_doc(d)
doc = mapping.get_mapped_record(doc)
try:
response_doc = connection.insert(mapping.remote_objectname, doc)
frappe.db.set_value(
mapping.local_doctype,
d.name,
mapping.migration_id_field,
response_doc[connection.name_field],
update_modified=False,
)
frappe.db.commit()
self.update_log("push_insert", 1)
# post process after insert
self.post_process_doc(local_doc=d, remote_doc=response_doc)
except Exception as e:
self.update_log("push_failed", {d.name: cstr(e)})
# update page_start
self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length)
if len(data) < mapping.page_length:
# done, no more new data to insert
self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0})
# not done with this mapping
return False
def _push_update(self):
"""Updates local modified docs on remote"""
mapping = self.get_mapping(self.current_mapping)
connection = self.get_connection()
data = self.get_updated_local_data()
for d in data:
migration_id_value = d.get(mapping.migration_id_field)
# pre process before update
doc = self.pre_process_doc(d)
doc = mapping.get_mapped_record(doc)
try:
response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value)
self.update_log("push_update", 1)
# post process after update
self.post_process_doc(local_doc=d, remote_doc=response_doc)
except Exception as e:
self.update_log("push_failed", {d.name: cstr(e)})
# update page_start
self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length)
if len(data) < mapping.page_length:
# done, no more data to update
self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0})
# not done with this mapping
return False
def _push_delete(self):
"""Deletes docs deleted from local on remote"""
mapping = self.get_mapping(self.current_mapping)
connection = self.get_connection()
data = self.get_deleted_local_data()
for d in data:
# Deleted Document also has a custom field for migration_id
migration_id_value = d.get(mapping.migration_id_field)
# pre process before update
self.pre_process_doc(d)
try:
response_doc = connection.delete(mapping.remote_objectname, migration_id_value)
self.update_log("push_delete", 1)
# post process only when action is success
self.post_process_doc(local_doc=d, remote_doc=response_doc)
except Exception as e:
self.update_log("push_failed", {d.name: cstr(e)})
# update page_start
self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length)
if len(data) < mapping.page_length:
# done, no more new data to delete
# done with this mapping
return True
def pull(self):
self.db_set("current_mapping_type", "Pull")
connection = self.get_connection()
mapping = self.get_mapping(self.current_mapping)
data = self.get_remote_data()
for d in data:
migration_id_value = get_source_value(d, connection.name_field)
doc = self.pre_process_doc(d)
doc = mapping.get_mapped_record(doc)
if migration_id_value:
try:
if not local_doc_exists(mapping, migration_id_value):
# insert new local doc
local_doc = insert_local_doc(mapping, doc)
self.update_log("pull_insert", 1)
# set migration id
frappe.db.set_value(
mapping.local_doctype,
local_doc.name,
mapping.migration_id_field,
migration_id_value,
update_modified=False,
)
frappe.db.commit()
else:
# update doc
local_doc = update_local_doc(mapping, doc, migration_id_value)
self.update_log("pull_update", 1)
# post process doc after success
self.post_process_doc(remote_doc=d, local_doc=local_doc)
except Exception as e:
# failed, append to log
self.update_log("pull_failed", {migration_id_value: cstr(e)})
if len(data) < mapping.page_length:
# last page, done with pull
return True
def pre_process_doc(self, doc):
plan = self.get_plan()
doc = plan.pre_process_doc(self.current_mapping, doc)
return doc
def post_process_doc(self, local_doc=None, remote_doc=None):
plan = self.get_plan()
doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc)
return doc
def set_log(self, key, value):
value = json.dumps(value) if "_failed" in key else value
self.db_set(key, value)
def update_log(self, key, value=None):
"""
Helper for updating logs,
push_failed and pull_failed are stored as json,
other keys are stored as int
"""
if "_failed" in key:
# json
self.set_log(key, self.get_log(key, []) + [value])
else:
# int
self.set_log(key, self.get_log(key, 0) + (value or 1))
def get_log(self, key, default=None):
value = self.db_get(key)
if "_failed" in key:
if not value:
value = json.dumps(default)
value = json.loads(value)
return value or default
def insert_local_doc(mapping, doc):
try:
# insert new doc
if not doc.doctype:
doc.doctype = mapping.local_doctype
doc = frappe.get_doc(doc).insert()
return doc
except Exception:
print("Data Migration Run failed: Error in Pull insert")
print(frappe.get_traceback())
return None
def update_local_doc(mapping, remote_doc, migration_id_value):
try:
# migration id value is set in migration_id_field in mapping.local_doctype
docname = frappe.db.get_value(
mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value}
)
doc = frappe.get_doc(mapping.local_doctype, docname)
doc.update(remote_doc)
doc.save()
return doc
except Exception:
print("Data Migration Run failed: Error in Pull update")
print(frappe.get_traceback())
return None
def local_doc_exists(mapping, migration_id_value):
return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value})

128
frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py
View file

@ -1,128 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# License: MIT. See LICENSE
import unittest
import frappe
class TestDataMigrationRun(unittest.TestCase):
def test_run(self):
create_plan()
description = "data migration todo"
new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert()
event_subject = "data migration event"
frappe.get_doc(
dict(
doctype="Event",
subject=event_subject,
repeat_on="Monthly",
starts_on=frappe.utils.now_datetime(),
)
).insert()
run = frappe.get_doc(
{
"doctype": "Data Migration Run",
"data_migration_plan": "ToDo Sync",
"data_migration_connector": "Local Connector",
}
).insert()
run.run()
self.assertEqual(run.db_get("status"), "Success")
self.assertEqual(run.db_get("push_insert"), 1)
self.assertEqual(run.db_get("pull_insert"), 1)
todo = frappe.get_doc("ToDo", new_todo.name)
self.assertTrue(todo.todo_sync_id)
# Pushed Event
event = frappe.get_doc("Event", todo.todo_sync_id)
self.assertEqual(event.subject, description)
# Pulled ToDo
created_todo = frappe.get_doc("ToDo", {"description": event_subject})
self.assertEqual(created_todo.description, event_subject)
todo_list = frappe.get_list(
"ToDo", filters={"description": "data migration todo"}, fields=["name"]
)
todo_name = todo_list[0].name
todo = frappe.get_doc("ToDo", todo_name)
todo.description = "data migration todo updated"
todo.save()
run = frappe.get_doc(
{
"doctype": "Data Migration Run",
"data_migration_plan": "ToDo Sync",
"data_migration_connector": "Local Connector",
}
).insert()
run.run()
# Update
self.assertEqual(run.db_get("status"), "Success")
self.assertEqual(run.db_get("pull_update"), 1)
def create_plan():
frappe.get_doc(
{
"doctype": "Data Migration Mapping",
"mapping_name": "Todo to Event",
"remote_objectname": "Event",
"remote_primary_key": "name",
"mapping_type": "Push",
"local_doctype": "ToDo",
"fields": [
{"remote_fieldname": "subject", "local_fieldname": "description"},
{
"remote_fieldname": "starts_on",
"local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())",
},
],
"condition": '{"description": "data migration todo" }',
}
).insert(ignore_if_duplicate=True)
frappe.get_doc(
{
"doctype": "Data Migration Mapping",
"mapping_name": "Event to ToDo",
"remote_objectname": "Event",
"remote_primary_key": "name",
"local_doctype": "ToDo",
"local_primary_key": "name",
"mapping_type": "Pull",
"condition": '{"subject": "data migration event" }',
"fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}],
}
).insert(ignore_if_duplicate=True)
frappe.get_doc(
{
"doctype": "Data Migration Plan",
"plan_name": "ToDo Sync",
"module": "Core",
"mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}],
}
).insert(ignore_if_duplicate=True)
frappe.get_doc(
{
"doctype": "Data Migration Connector",
"connector_name": "Local Connector",
"connector_type": "Frappe",
# connect to same host.
"hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site),
"username": "Administrator",
"password": frappe.conf.get("admin_password") or "admin",
}
).insert(ignore_if_duplicate=True)

18
frappe/model/sync.py
View file

@ -53,22 +53,6 @@ def sync_for(app_name, force=0, reset_permissions=False):
os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json")
)
for data_migration_module in [
"data_migration_mapping_detail",
"data_migration_mapping",
"data_migration_plan_mapping",
"data_migration_plan",
]:
files.append(
os.path.join(
FRAPPE_PATH,
"data_migration",
"doctype",
data_migration_module,
f"{data_migration_module}.json",
)
)
for desk_module in [
"number_card",
"dashboard_chart",
@ -124,8 +108,6 @@ def get_doc_files(files, start_path):
"web_template",
"notification",
"print_style",
"data_migration_mapping",
"data_migration_plan",
"workspace",
"onboarding_step",
"module_onboarding",

1
frappe/modules.txt
View file

@ -8,7 +8,6 @@ Desk
Integrations
Printing
Contacts
Data Migration
Social
Automation
Event Streaming

1
frappe/patches.txt
View file

@ -201,3 +201,4 @@ frappe.patches.v14_0.update_color_names_in_kanban_board_column
frappe.patches.v14_0.update_is_system_generated_flag
frappe.patches.v14_0.update_auto_account_deletion_duration
frappe.patches.v14_0.set_document_expiry_default
frappe.patches.v14_0.delete_data_migration_tool

12
frappe/patches/v14_0/delete_data_migration_tool.py Normal file
View file

@ -0,0 +1,12 @@
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
import frappe
def execute():
doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name")
for doctype in doctypes:
frappe.delete_doc("DocType", doctype, ignore_missing=True)
frappe.delete_doc("Module Def", "Data Migration", ignore_missing=True, force=True)