From 739b03348955e643ddb0fd4c53ae78aff8be3e4c Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Wed, 14 Aug 2019 13:22:27 +0530 Subject: [PATCH] feat: implemented pull trigger in follower nodes --- frappe/modules/patch_handler.py | 5 +- frappe/offline/doctype/node/node.json | 18 ++-- frappe/offline/doctype/node/node.py | 1 - .../node_configuration/node_configuration.js | 2 +- .../node_configuration/node_configuration.py | 86 ++++++++++++------- .../offline/doctype/update_log/update_log.py | 10 +-- .../public/js/frappe/ui/toolbar/navbar.html | 2 + frappe/public/js/frappe/ui/toolbar/toolbar.js | 20 +++++ 8 files changed, 97 insertions(+), 47 deletions(-) diff --git a/frappe/modules/patch_handler.py b/frappe/modules/patch_handler.py index b3237b8b76..44ed3148db 100644 --- a/frappe/modules/patch_handler.py +++ b/frappe/modules/patch_handler.py @@ -119,11 +119,12 @@ def executed(patchmodule): # print "Patch %s already executed in %s" % (patchmodule, frappe.db.cur_db_name) return done -def block_user(block): +def block_user(block, msg = None): """stop/start execution till patch is run""" frappe.local.flags.in_patch = block frappe.db.begin() - msg = "Patches are being executed in the system. Please try again in a few moments." + if not msg: + msg = "Patches are being executed in the system. Please try again in a few moments." frappe.db.set_global('__session_status', block and 'stop' or None) frappe.db.set_global('__session_status_message', block and msg or None) frappe.db.commit() diff --git a/frappe/offline/doctype/node/node.json b/frappe/offline/doctype/node/node.json index 795333e146..66dd09e04a 100644 --- a/frappe/offline/doctype/node/node.json +++ b/frappe/offline/doctype/node/node.json @@ -1,6 +1,4 @@ { - "_comments": "[]", - "_liked_by": "[]", "autoname": "field:host_name", "creation": "2019-07-30 15:10:47.993692", "doctype": "DocType", @@ -9,6 +7,7 @@ "host_name", "api_key", "api_secret", + "user", "allow_auto_changes", "last_updated" ], @@ -16,7 +15,9 @@ { "fieldname": "host_name", "fieldtype": "Data", + "in_list_view": 1, "label": "Hostname (URL)", + "reqd": 1, "unique": 1 }, { @@ -43,11 +44,18 @@ "fieldname": "last_updated", "fieldtype": "Link", "label": "Last Updated", - "options": "Update Log", - "read_only": 1 + "options": "Update Log" + }, + { + "fieldname": "user", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Node User", + "options": "User", + "reqd": 1 } ], - "modified": "2019-08-09 14:54:07.173962", + "modified": "2019-08-14 10:19:30.049821", "modified_by": "Administrator", "module": "Offline", "name": "Node", diff --git a/frappe/offline/doctype/node/node.py b/frappe/offline/doctype/node/node.py index 1553a9c5a2..c8fb58d217 100644 --- a/frappe/offline/doctype/node/node.py +++ b/frappe/offline/doctype/node/node.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import frappe -from frappe.frappeclient import FrappeClient from frappe.model.document import Document class Node(Document): diff --git a/frappe/offline/doctype/node_configuration/node_configuration.js b/frappe/offline/doctype/node_configuration/node_configuration.js index cf979bb791..68f7dd0b7b 100644 --- a/frappe/offline/doctype/node_configuration/node_configuration.js +++ b/frappe/offline/doctype/node_configuration/node_configuration.js @@ -3,6 +3,6 @@ frappe.ui.form.on('Node Configuration', { // refresh: function(frm) { - // + // } }); diff --git a/frappe/offline/doctype/node_configuration/node_configuration.py b/frappe/offline/doctype/node_configuration/node_configuration.py index 371c121f7d..d28c9b9936 100644 --- a/frappe/offline/doctype/node_configuration/node_configuration.py +++ b/frappe/offline/doctype/node_configuration/node_configuration.py @@ -8,60 +8,86 @@ from frappe import _ import json from frappe.model.document import Document from frappe.frappeclient import FrappeClient +from frappe.model.document import check_doctype_has_followers +from frappe.custom.doctype.custom_field.custom_field import create_custom_field class NodeConfiguration(Document): def before_insert(self): config_exists = frappe.db.get_all( - doctype = 'Node Configuration', - filters = [ - ['master_node', '=', self.master_node], - ['follower_node', '=', self.follower_node] - ] + doctype = 'Node Configuration', + filters = { + 'master_node': self.master_node, + 'follower_node': self.follower_node + } ) if config_exists: - frappe.throw(_('Node Configuration already exists')) + frappe.throw(_('Node configuration already exists')) + + def on_update(self): + ''' create custom field to store remote docname of master node''' + df = { + 'label': 'Remote Docname', + 'fieldname': 'remote_docname', + 'fieldtype': 'Data', + 'hidden': 1, + 'read_only': 1, + 'unique': 1, + 'no_copy': 1 + } + for doc in self.following_doctypes: + print(doc) + create_custom_field(doc.ref_doctype, df) @frappe.whitelist() -def sync_master_data(): - '''Sync master data to all follower nodes, triggered when update log is created''' +def pull_master_data(): + '''Fetch data from remote master node.''' current_node = frappe.utils.get_url() - port = frappe.conf.http_port or frappe.conf.webserver_port - current_node = current_node + ':' + str(port) - node_configurations = frappe.get_all( doctype = 'Node Configuration', - filters = {'master_node': current_node}, - group_by = 'follower_node' + filters = {'follower_node': current_node} ) - for node_config in node_configurations: config = frappe.get_doc('Node Configuration', node_config.name) + client = FrappeClient(config.master_node, 'Administrator', 'root') + remote_node = client.get_doc('Node', filters = {'host_name': current_node}, fields = ['name', 'last_updated']) + last_update_synced = client.get_value('Update Log', 'creation', filters = {'name': remote_node[0].get('last_updated')}) - last_updated = frappe.db.get_value('Node', config.follower_node, 'last_updated') - last_update_synced = frappe.db.get_value('Update Log', last_updated, 'creation') - doctypes = [] for entry in config.following_doctypes: doctypes.append(entry.ref_doctype) - updates_to_be_synced = frappe.get_all( + updates_to_be_synced = client.get_list( doctype = 'Update Log', filters = [['creation', '>', last_update_synced], ['ref_doctype', 'in', doctypes]], fields = ['update_type', 'ref_doctype', 'docname', 'data', 'name'], - order_by = 'creation' ) if updates_to_be_synced != []: - client = FrappeClient(config.follower_node, 'Administrator', 'root') - for doc in updates_to_be_synced: - if doc.update_type == 'Create': - client.insert(json.loads(doc.data)) - elif doc.update_type == 'Update': - client.update(json.loads(doc.data)) - elif doc.update == 'Delete': - client.delete(doc.ref_doctype, doc.docname) + try: + if doc.get('update_type') == 'Create': + local_doc = frappe.get_doc(json.loads(doc.get('data'))).insert() + doc = frappe.db.set_value(doc.get('ref_doctype'), local_doc.name, 'remote_docname', doc.get('name')) - #set the last update for node - frappe.db.set_value('Node', config.follower_node, 'last_updated', updates_to_be_synced[-1].get('name')) - \ No newline at end of file + if doc.get('update_type') == 'Update': + mapped_doc = frappe.get_all(doc.get('ref_doctype'), filters = {'remote_docname': doc.get('name')}, fields = ['name']) + local_doc = frappe.get_doc(doc.get('doctype'), mapped_doc[0].get('name')) + local_doc.update(json.loads(doc.get('data'))) + + if doc.get('update_type') == 'Delete': + mapped_doc = frappe.get_all(doc.get('ref_doctype'), filters = {'remote_docname': doc.get('name')}, fields = ['name']) + local_doc = frappe.get_doc(doc.get('doctype'), mapped_doc[0].get('name')) + local_doc.delete() + frappe.db.commit() + + except Exception: + frappe.db.rollback() + frappe.log_error(frappe.get_traceback(), _('Pulling master data failed')) + return 'failed' + + client.set_value('Node', remote_node[0].get('name'), 'last_updated', updates_to_be_synced[-1].name) + + else: + return 'no updates' + + return 'success' \ No newline at end of file diff --git a/frappe/offline/doctype/update_log/update_log.py b/frappe/offline/doctype/update_log/update_log.py index 5bb2bd3061..98c8080250 100644 --- a/frappe/offline/doctype/update_log/update_log.py +++ b/frappe/offline/doctype/update_log/update_log.py @@ -3,14 +3,8 @@ # For license information, please see license.txt from __future__ import unicode_literals -import frappe, json +import frappe from frappe.model.document import Document -from frappe.utils.background_jobs import get_jobs class UpdateLog(Document): - def validate(self): - '''Sync master data to followers whenever update log is generated''' - enqueued_method = 'frappe.offline.doctype.node_configuration.node_configuration.sync_master_data' - jobs = get_jobs() - if not jobs or enqueued_method not in jobs[frappe.local.site]: - frappe.enqueue(enqueued_method, queue = 'default') + pass \ No newline at end of file diff --git a/frappe/public/js/frappe/ui/toolbar/navbar.html b/frappe/public/js/frappe/ui/toolbar/navbar.html index 1508fb1cb7..5cdf790f6f 100644 --- a/frappe/public/js/frappe/ui/toolbar/navbar.html +++ b/frappe/public/js/frappe/ui/toolbar/navbar.html @@ -41,6 +41,8 @@
  • {%= __("Background Jobs") %}
  • +
  • + {%= __("Pull Master Data") %}
  • {%= __("Logout") %}
    • diff --git a/frappe/public/js/frappe/ui/toolbar/toolbar.js b/frappe/public/js/frappe/ui/toolbar/toolbar.js index 9bca9275b6..c3f487058c 100644 --- a/frappe/public/js/frappe/ui/toolbar/toolbar.js +++ b/frappe/public/js/frappe/ui/toolbar/toolbar.js @@ -300,3 +300,23 @@ frappe.ui.toolbar.setup_session_defaults = function() { } }); }; + +frappe.ui.toolbar.pull_master_data = function() { + frappe.call({ + method: 'frappe.offline.doctype.node_configuration.node_configuration.pull_master_data', + callback: function(r) { + if(r.message == 'success') { + frappe.show_alert({message:'Successfully pulled master data', indicator:'green'}); + location.reload(true); + } + else if(r.message == 'failed') { + frappe.show_alert({message:'Failed to pull master data', indicator:'red'}); + location.reload(true); + } + else if(r.message == 'no updates') { + frappe.show_alert({message:'Already up-to-date', indicator:'blue'}); + location.reload(true); + } + } + }); +}