From 11fca9fb5c7ace00ef7b05ad9e6cbe7435dfb31e Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Mon, 22 Apr 2019 12:55:13 +0530 Subject: [PATCH] feat: Maintain list of tables touched during migrate This can be used to selectively restore changed tables from backup after migrate failure --- frappe/database.py | 22 +++++++++++++ frappe/migrate.py | 70 ++++++++++++++++++++++++----------------- frappe/tests/test_db.py | 29 +++++++++++++++++ 3 files changed, 93 insertions(+), 28 deletions(-) diff --git a/frappe/database.py b/frappe/database.py index 10a51af24e..9323aa9518 100644 --- a/frappe/database.py +++ b/frappe/database.py @@ -197,6 +197,10 @@ class Database: frappe.log(values) frappe.log(">>>>") self._cursor.execute(query, values) + + if frappe.flags.in_migrate: + self.log_touched_tables(query, values) + else: if debug: if explain: @@ -209,6 +213,9 @@ class Database: self._cursor.execute(query) + if frappe.flags.in_migrate: + self.log_touched_tables(query) + if debug: time_end = time() frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2))) @@ -976,6 +983,21 @@ class Database: # when document does not exist return [] + def log_touched_tables(self, query, values=None): + if values: + query = self._cursor.mogrify(query, values) + if query.strip().lower().split()[0] in ('insert', 'delete', 'update', 'alter'): + # ([`\"']?) Captures ', " or ` at the begining of the table name (if provided) + # (tab([A-Z]\w+)( [A-Z]\w+)*) Captures table names that start with "tab" + # and are continued with multiple words that start with a captital letter + # e.g. 'tabXxx' or 'tabXxx Xxx' or 'tabXxx Xxx Xxx' and so on + # \1 matches the first captured group (quote character) at the end of the table name + tables = [groups[1] for groups in re.findall(r'([`"\']?)(tab([A-Z]\w+)( [A-Z]\w+)*)\1', query)] + if frappe.flags.touched_tables is None: + frappe.flags.touched_tables = set() + frappe.flags.touched_tables.update(tables) + + def enqueue_jobs_after_commit(): if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0: for job in frappe.flags.enqueue_after_commit: diff --git a/frappe/migrate.py b/frappe/migrate.py index 956b4a3c93..1952c3cd6f 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals +import json +import os import frappe import frappe.translate import frappe.modules.patch_handler @@ -26,40 +28,52 @@ def migrate(verbose=True, rebuild_website=False): - sync web pages (from /www) - run after migrate hooks ''' - frappe.flags.in_migrate = True - clear_global_cache() - #run before_migrate hooks - for app in frappe.get_installed_apps(): - for fn in frappe.get_hooks('before_migrate', app_name=app): - frappe.get_attr(fn)() + touched_tables_file = frappe.get_site_path('touched_tables.json') + if os.path.exists(touched_tables_file): + os.remove(touched_tables_file) - # run patches - frappe.modules.patch_handler.run_all() - # sync - frappe.model.sync.sync_all(verbose=verbose) - frappe.translate.clear_cache() - sync_fixtures() - sync_customizations() - sync_desktop_icons() - sync_languages() + try: + frappe.flags.touched_tables = set() + frappe.flags.in_migrate = True + clear_global_cache() - frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu() + #run before_migrate hooks + for app in frappe.get_installed_apps(): + for fn in frappe.get_hooks('before_migrate', app_name=app): + frappe.get_attr(fn)() - # syncs statics - render.clear_cache() + # run patches + frappe.modules.patch_handler.run_all() + # sync + frappe.model.sync.sync_all(verbose=verbose) + frappe.translate.clear_cache() + sync_fixtures() + sync_customizations() + sync_desktop_icons() + sync_languages() - # add static pages to global search - router.sync_global_search() + frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu() - #run after_migrate hooks - for app in frappe.get_installed_apps(): - for fn in frappe.get_hooks('after_migrate', app_name=app): - frappe.get_attr(fn)() + # syncs statics + render.clear_cache() - frappe.db.commit() + # add static pages to global search + router.sync_global_search() - clear_notifications() + #run after_migrate hooks + for app in frappe.get_installed_apps(): + for fn in frappe.get_hooks('after_migrate', app_name=app): + frappe.get_attr(fn)() + + frappe.db.commit() + + clear_notifications() + + frappe.publish_realtime("version-update") + frappe.flags.in_migrate = False + finally: + with open(touched_tables_file, 'w') as f: + json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) + frappe.flags.touched_tables.clear() - frappe.publish_realtime("version-update") - frappe.flags.in_migrate = False diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index ea6c07ae76..e3c1a62a78 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals import unittest import frappe +from frappe.custom.doctype.custom_field.custom_field import create_custom_field class TestDB(unittest.TestCase): def test_get_value(self): @@ -27,3 +28,31 @@ class TestDB(unittest.TestCase): # def test_multiple_queries(self): # # implicit commit # self.assertRaises(frappe.SQLError, frappe.db.sql, """select name from `tabUser`; truncate `tabEmail Queue`""") + + def test_log_touched_tables(self): + frappe.flags.in_migrate = True + frappe.flags.touched_tables = set() + frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5) + self.assertIn('tabSingles', frappe.flags.touched_tables) + + frappe.flags.touched_tables = set() + todo = frappe.get_doc({'doctype': 'ToDo', 'description': 'Random Description'}) + todo.save() + self.assertIn('tabToDo', frappe.flags.touched_tables) + + frappe.flags.touched_tables = set() + todo.description = "Another Description" + todo.save() + self.assertIn('tabToDo', frappe.flags.touched_tables) + + frappe.flags.touched_tables = set() + todo.delete() + self.assertIn('tabToDo', frappe.flags.touched_tables) + + frappe.flags.touched_tables = set() + create_custom_field('ToDo', {'label': 'ToDo Custom Field'}) + + self.assertIn('tabToDo', frappe.flags.touched_tables) + self.assertIn('tabCustom Field', frappe.flags.touched_tables) + frappe.flags.in_migrate = False + frappe.flags.touched_tables.clear()