Merge pull request #7334 from adityahase/feat-touched-tables-hotfix

feat: Maintain list of tables touched during migrate

frappe/database.py

@@ -197,6 +197,10 @@ class Database:
 					frappe.log(values)
 					frappe.log(">>>>")
 				self._cursor.execute(query, values)
+
+				if frappe.flags.in_migrate:
+					self.log_touched_tables(query, values)
+
 			else:
 				if debug:
 					if explain:
@@ -209,6 +213,9 @@ class Database:
 
 				self._cursor.execute(query)
 
+				if frappe.flags.in_migrate:
+					self.log_touched_tables(query)
+
 			if debug:
 				time_end = time()
 				frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2)))
@@ -976,6 +983,21 @@ class Database:
 			# when document does not exist
 			return []
 
+	def log_touched_tables(self, query, values=None):
+		if values:
+			query = self._cursor.mogrify(query, values)
+		if query.strip().lower().split()[0] in ('insert', 'delete', 'update', 'alter'):
+			# ([`\"']?) Captures ', " or ` at the begining of the table name (if provided)
+			# (tab([A-Z]\w+)( [A-Z]\w+)*) Captures table names that start with "tab"
+			# and are continued with multiple words that start with a captital letter
+			# e.g. 'tabXxx' or 'tabXxx Xxx' or 'tabXxx Xxx Xxx' and so on
+			# \1 matches the first captured group (quote character) at the end of the table name
+			tables = [groups[1] for groups in re.findall(r'([`"\']?)(tab([A-Z]\w+)( [A-Z]\w+)*)\1', query)]
+			if frappe.flags.touched_tables is None:
+				frappe.flags.touched_tables = set()
+			frappe.flags.touched_tables.update(tables)
+
+
 def enqueue_jobs_after_commit():
 	if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0:
 		for job in frappe.flags.enqueue_after_commit:

frappe/migrate.py

@@ -3,6 +3,8 @@
 
 from __future__ import unicode_literals
 
+import json
+import os
 import frappe
 import frappe.translate
 import frappe.modules.patch_handler
@@ -26,40 +28,52 @@ def migrate(verbose=True, rebuild_website=False):
 	- sync web pages (from /www)
 	- run after migrate hooks
 	'''
-	frappe.flags.in_migrate = True
-	clear_global_cache()
 
-	#run before_migrate hooks
-	for app in frappe.get_installed_apps():
-		for fn in frappe.get_hooks('before_migrate', app_name=app):
-			frappe.get_attr(fn)()
+	touched_tables_file = frappe.get_site_path('touched_tables.json')
+	if os.path.exists(touched_tables_file):
+		os.remove(touched_tables_file)
 
-	# run patches
-	frappe.modules.patch_handler.run_all()
-	# sync
-	frappe.model.sync.sync_all(verbose=verbose)
-	frappe.translate.clear_cache()
-	sync_fixtures()
-	sync_customizations()
-	sync_desktop_icons()
-	sync_languages()
+	try:
+		frappe.flags.touched_tables = set()
+		frappe.flags.in_migrate = True
+		clear_global_cache()
 
-	frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu()
+		#run before_migrate hooks
+		for app in frappe.get_installed_apps():
+			for fn in frappe.get_hooks('before_migrate', app_name=app):
+				frappe.get_attr(fn)()
 
-	# syncs statics
-	render.clear_cache()
+		# run patches
+		frappe.modules.patch_handler.run_all()
+		# sync
+		frappe.model.sync.sync_all(verbose=verbose)
+		frappe.translate.clear_cache()
+		sync_fixtures()
+		sync_customizations()
+		sync_desktop_icons()
+		sync_languages()
 
-	# add static pages to global search
-	router.sync_global_search()
+		frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu()
 
-	#run after_migrate hooks
-	for app in frappe.get_installed_apps():
-		for fn in frappe.get_hooks('after_migrate', app_name=app):
-			frappe.get_attr(fn)()
+		# syncs statics
+		render.clear_cache()
 
-	frappe.db.commit()
+		# add static pages to global search
+		router.sync_global_search()
 
-	clear_notifications()
+		#run after_migrate hooks
+		for app in frappe.get_installed_apps():
+			for fn in frappe.get_hooks('after_migrate', app_name=app):
+				frappe.get_attr(fn)()
+
+		frappe.db.commit()
+
+		clear_notifications()
+
+		frappe.publish_realtime("version-update")
+		frappe.flags.in_migrate = False
+	finally:
+		with open(touched_tables_file, 'w') as f:
+			json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4)
+		frappe.flags.touched_tables.clear()
 
-	frappe.publish_realtime("version-update")
-	frappe.flags.in_migrate = False

frappe/tests/test_db.py

@@ -6,6 +6,7 @@
 from __future__ import unicode_literals
 import unittest
 import frappe
+from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 
 class TestDB(unittest.TestCase):
 	def test_get_value(self):
@@ -27,3 +28,31 @@ class TestDB(unittest.TestCase):
 	# def test_multiple_queries(self):
 	# 	# implicit commit
 	# 	self.assertRaises(frappe.SQLError, frappe.db.sql, """select name from `tabUser`; truncate `tabEmail Queue`""")
+
+	def test_log_touched_tables(self):
+		frappe.flags.in_migrate = True
+		frappe.flags.touched_tables = set()
+		frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5)
+		self.assertIn('tabSingles', frappe.flags.touched_tables)
+
+		frappe.flags.touched_tables = set()
+		todo = frappe.get_doc({'doctype': 'ToDo', 'description': 'Random Description'})
+		todo.save()
+		self.assertIn('tabToDo', frappe.flags.touched_tables)
+
+		frappe.flags.touched_tables = set()
+		todo.description = "Another Description"
+		todo.save()
+		self.assertIn('tabToDo', frappe.flags.touched_tables)
+
+		frappe.flags.touched_tables = set()
+		todo.delete()
+		self.assertIn('tabToDo', frappe.flags.touched_tables)
+
+		frappe.flags.touched_tables = set()
+		create_custom_field('ToDo', {'label': 'ToDo Custom Field'})
+
+		self.assertIn('tabToDo', frappe.flags.touched_tables)
+		self.assertIn('tabCustom Field', frappe.flags.touched_tables)
+		frappe.flags.in_migrate = False
+		frappe.flags.touched_tables.clear()