feat: master-follower one way sync

frappe/model/delete_doc.py

@@ -18,7 +18,8 @@ from frappe.model.naming import revert_series_if_last
 from frappe.utils.global_search import delete_for_document
 from frappe.desk.doctype.tag.tag import delete_tags_for_document
 from frappe.exceptions import FileNotFoundError
-
+from six import string_types, integer_types
+from frappe.model.document import make_update_log
 
 doctypes_to_skip = ("Communication", "ToDo", "DocShare", "Email Unsubscribe", "Activity Log", "File", "Version", "Document Follow", "Comment" , "View Log", "Tag Link")
 
@@ -120,6 +121,10 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa
 		# delete tag link entry
 		delete_tags_for_document(doc)
 
+		# update log if doctype has followers
+		if not frappe.flags.in_install:
+			make_update_log(doc, update_type = 'Delete')
+
 		if doc and not for_reload:
 			add_to_deleted_document(doc)
 			if not frappe.flags.in_patch:

frappe/model/document.py

@@ -255,6 +255,8 @@ class Document(BaseDocument):
 		if self.get("amended_from"):
 			self.copy_attachments_from_amended_from()
 
+		#flag to prevent creation of update log for create and update both, during document creation
+		self.flags.update_log_for_doc_creation = True
 		self.run_post_save_methods()
 		self.flags.in_insert = False
 
@@ -937,6 +939,14 @@ class Document(BaseDocument):
 		if (self.doctype, self.name) in frappe.flags.currently_saving:
 			frappe.flags.currently_saving.remove((self.doctype, self.name))
 
+		# make update log for doctypes having followers
+		if not frappe.flags.in_install:
+			if self.flags.update_log_for_doc_creation:
+				make_update_log(self, update_type = 'Create')
+				self.flags.create_type_update_log = False
+			else:
+				make_update_log(self, update_type = 'Update')
+
 		self.latest = None
 
 	def clear_cache(self):
@@ -1261,3 +1271,30 @@ def execute_action(doctype, name, action, **kwargs):
 
 		doc.add_comment('Comment', _('Action Failed') + '<br><br>' + msg)
 		doc.notify_update()
+
+def make_update_log(doc, update_type):
+	'''Save update info for doctypes that have followers'''
+	doctype_has_followers = check_doctype_has_followers(doc.doctype)
+	if doctype_has_followers:
+		if update_type != 'Delete':
+			data = frappe.as_json(doc)
+		else:
+			data = None
+		doc = frappe.get_doc({
+			'doctype': 'Update Log',
+			'update_type': update_type,
+			'ref_doctype': doc.doctype,
+			'docname': doc.name,
+			'data': data
+		})
+		doc.insert(ignore_permissions = True)
+		frappe.db.commit()
+
+def check_doctype_has_followers(doctype):
+	node_configs = frappe.get_all(doctype = 'Node Configuration')
+	for node_config in node_configs:
+		config = frappe.get_doc('Node Configuration', node_config.name)
+		for entry in config.following_doctypes:
+			if doctype == entry.ref_doctype:
+				return True
+	return False

frappe/modules.txt

@@ -12,3 +12,4 @@ Data Migration
 Chat
 Social
 Automation
+Offline

frappe/offline/doctype/change_request/change_request.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Change Request', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/offline/doctype/change_request/change_request.json

@@ -0,0 +1,65 @@
+{
+ "creation": "2019-07-30 16:10:05.784349",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "follower_node",
+  "ref_doctype",
+  "document_name",
+  "requested_change",
+  "request_status"
+ ],
+ "fields": [
+  {
+   "fieldname": "follower_node",
+   "fieldtype": "Link",
+   "label": "Follower Node",
+   "options": "Node"
+  },
+  {
+   "fieldname": "ref_doctype",
+   "fieldtype": "Data",
+   "label": "DocType"
+  },
+  {
+   "fieldname": "document_name",
+   "fieldtype": "Data",
+   "label": "Document Name"
+  },
+  {
+   "fieldname": "requested_change",
+   "fieldtype": "Code",
+   "label": "Requested Change"
+  },
+  {
+   "fieldname": "request_status",
+   "fieldtype": "Select",
+   "label": "Request Status",
+   "options": "\nApproved\nRejected\nResolve Conflict"
+  }
+ ],
+ "modified": "2019-08-02 09:48:41.263955",
+ "modified_by": "Administrator",
+ "module": "Offline",
+ "name": "Change Request",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/offline/doctype/change_request/change_request.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class ChangeRequest(Document):
+	pass

frappe/offline/doctype/change_request/test_change_request.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestChangeRequest(unittest.TestCase):
+	pass

frappe/offline/doctype/node/node.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Node', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/offline/doctype/node/node.json

@@ -0,0 +1,73 @@
+{
+ "_comments": "[]",
+ "_liked_by": "[]",
+ "autoname": "field:host_name",
+ "creation": "2019-07-30 15:10:47.993692",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "host_name",
+  "api_key",
+  "api_secret",
+  "allow_auto_changes",
+  "last_updated"
+ ],
+ "fields": [
+  {
+   "fieldname": "host_name",
+   "fieldtype": "Data",
+   "label": "Hostname (URL)",
+   "unique": 1
+  },
+  {
+   "default": "0",
+   "description": "Automatic changes from Follower nodes will be fetched and updated",
+   "fieldname": "allow_auto_changes",
+   "fieldtype": "Check",
+   "label": "Allow Automatic Changes"
+  },
+  {
+   "fieldname": "api_key",
+   "fieldtype": "Data",
+   "label": "API Key",
+   "read_only": 1,
+   "unique": 1
+  },
+  {
+   "fieldname": "api_secret",
+   "fieldtype": "Password",
+   "label": "API Secret",
+   "read_only": 1
+  },
+  {
+   "fieldname": "last_updated",
+   "fieldtype": "Link",
+   "label": "Last Updated",
+   "options": "Update Log",
+   "read_only": 1
+  }
+ ],
+ "modified": "2019-08-09 14:54:07.173962",
+ "modified_by": "Administrator",
+ "module": "Offline",
+ "name": "Node",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/offline/doctype/node/node.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.frappeclient import FrappeClient
+from frappe.model.document import Document
+
+class Node(Document):
+	def before_insert(self):
+		self.api_key = frappe.generate_hash(length = 15)
+		self.api_secret = frappe.generate_hash(length = 15)

frappe/offline/doctype/node/test_node.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestNode(unittest.TestCase):
+	pass

frappe/offline/doctype/node_configuration/node_configuration.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Node Configuration', {
+	// refresh: function(frm) {
+  //
+	// }
+});

frappe/offline/doctype/node_configuration/node_configuration.json

@@ -0,0 +1,85 @@
+{
+ "autoname": "format:NODE-CONFIG-{#####}",
+ "creation": "2019-07-30 15:39:28.765991",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "master_node",
+  "following_doctypes",
+  "follower_node",
+  "rules_section",
+  "allow_creation",
+  "allow_update",
+  "allow_deletion"
+ ],
+ "fields": [
+  {
+   "fieldname": "follower_node",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Follower Node",
+   "options": "Node",
+   "reqd": 1
+  },
+  {
+   "fieldname": "master_node",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Master Node",
+   "options": "Node",
+   "reqd": 1
+  },
+  {
+   "fieldname": "rules_section",
+   "fieldtype": "Section Break",
+   "label": "Rules for Follower"
+  },
+  {
+   "default": "0",
+   "fieldname": "allow_creation",
+   "fieldtype": "Check",
+   "label": "Allow Creation"
+  },
+  {
+   "default": "0",
+   "fieldname": "allow_update",
+   "fieldtype": "Check",
+   "label": "Allow Update"
+  },
+  {
+   "default": "0",
+   "fieldname": "allow_deletion",
+   "fieldtype": "Check",
+   "label": "Allow Deletion"
+  },
+  {
+   "fieldname": "following_doctypes",
+   "fieldtype": "Table",
+   "label": "Following DocTypes",
+   "options": "Node Configuration DocType",
+   "reqd": 1
+  }
+ ],
+ "modified": "2019-08-11 22:35:18.561804",
+ "modified_by": "Administrator",
+ "module": "Offline",
+ "name": "Node Configuration",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/offline/doctype/node_configuration/node_configuration.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe import _
+import json
+from frappe.model.document import Document
+from frappe.frappeclient import FrappeClient
+
+class NodeConfiguration(Document):
+	def before_insert(self):
+		config_exists = frappe.db.get_all(
+			doctype = 'Node Configuration', 
+			filters = [
+				['master_node', '=', self.master_node],
+				['follower_node', '=', self.follower_node]
+			]
+		)
+		if config_exists:
+			frappe.throw(_('Node Configuration already exists'))
+
+@frappe.whitelist()
+def sync_master_data():
+	'''Sync master data to all follower nodes, triggered when update log is created'''
+	current_node = frappe.utils.get_url()
+	port = frappe.conf.http_port or frappe.conf.webserver_port
+	current_node = current_node + ':' + str(port)
+
+	node_configurations = frappe.get_all(
+		doctype = 'Node Configuration',
+		filters = {'master_node': current_node},
+		group_by = 'follower_node'
+	)
+
+	for node_config in node_configurations:
+		config = frappe.get_doc('Node Configuration', node_config.name)
+
+		last_updated = frappe.db.get_value('Node', config.follower_node, 'last_updated')
+		last_update_synced = frappe.db.get_value('Update Log', last_updated, 'creation')
+		
+		doctypes = []
+		for entry in config.following_doctypes:
+			doctypes.append(entry.ref_doctype)
+
+		updates_to_be_synced = frappe.get_all(
+			doctype = 'Update Log',
+			filters = [['creation', '>', last_update_synced], ['ref_doctype', 'in', doctypes]],
+			fields = ['update_type', 'ref_doctype', 'docname', 'data', 'name'],
+			order_by = 'creation'
+		)
+
+		if updates_to_be_synced != []:
+			client = FrappeClient(config.follower_node, 'Administrator', 'root')
+			
+			for doc in updates_to_be_synced:
+				if doc.update_type == 'Create':
+					client.insert(json.loads(doc.data))
+				elif doc.update_type == 'Update':
+					client.update(json.loads(doc.data))
+				elif doc.update == 'Delete':
+					client.delete(doc.ref_doctype, doc.docname)
+
+			#set the last update for node
+			frappe.db.set_value('Node', config.follower_node, 'last_updated', updates_to_be_synced[-1].get('name'))
+			

frappe/offline/doctype/node_configuration/test_node_configuration.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import frappe
+import unittest
+
+class TestNodeConfiguration(unittest.TestCase):
+	pass

frappe/offline/doctype/node_configuration_doctype/node_configuration_doctype.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2019, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Node Configuration DocType', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/offline/doctype/node_configuration_doctype/node_configuration_doctype.json

@@ -0,0 +1,30 @@
+{
+ "creation": "2019-08-09 16:19:58.344278",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "ref_doctype"
+ ],
+ "fields": [
+  {
+   "fieldname": "ref_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Document Type",
+   "options": "DocType",
+   "reqd": 1
+  }
+ ],
+ "istable": 1,
+ "modified": "2019-08-09 16:21:12.082995",
+ "modified_by": "Administrator",
+ "module": "Offline",
+ "name": "Node Configuration DocType",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/offline/doctype/node_configuration_doctype/node_configuration_doctype.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class NodeConfigurationDocType(Document):
+	pass

frappe/offline/doctype/node_configuration_doctype/test_node_configuration_doctype.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestNodeConfigurationDocType(unittest.TestCase):
+	pass

frappe/offline/doctype/update_log/test_update_log.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestUpdateLog(unittest.TestCase):
+	pass

frappe/offline/doctype/update_log/update_log.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Update Log', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/offline/doctype/update_log/update_log.json

@@ -0,0 +1,65 @@
+{
+ "creation": "2019-07-30 15:31:26.352527",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "update_type",
+  "ref_doctype",
+  "docname",
+  "data"
+ ],
+ "fields": [
+  {
+   "fieldname": "update_type",
+   "fieldtype": "Select",
+   "label": "Update Type",
+   "options": "Create\nUpdate\nDelete",
+   "read_only": 1
+  },
+  {
+   "fieldname": "ref_doctype",
+   "fieldtype": "Link",
+   "label": "DocType",
+   "options": "DocType",
+   "read_only": 1
+  },
+  {
+   "fieldname": "docname",
+   "fieldtype": "Data",
+   "label": "Document Name",
+   "options": "ref_doctype",
+   "read_only": 1
+  },
+  {
+   "fieldname": "data",
+   "fieldtype": "Code",
+   "label": "Data",
+   "read_only": 1
+  }
+ ],
+ "in_create": 1,
+ "modified": "2019-08-04 23:28:11.548894",
+ "modified_by": "Administrator",
+ "module": "Offline",
+ "name": "Update Log",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/offline/doctype/update_log/update_log.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe, json
+from frappe.model.document import Document
+from frappe.utils.background_jobs import get_jobs
+
+class UpdateLog(Document):
+	def validate(self):
+		'''Sync master data to followers whenever update log is generated'''
+		enqueued_method = 'frappe.offline.doctype.node_configuration.node_configuration.sync_master_data'
+		jobs = get_jobs()
+		if not jobs or enqueued_method not in jobs[frappe.local.site]:
+			frappe.enqueue(enqueued_method, queue = 'default')