Merge pull request #14246 from saxenabhishek/aks-fix-migration

feat: Hash based comparison migration
This commit is contained in:
gavin 2021-10-12 17:50:49 +05:30 committed by GitHub
commit 8e7d83c88d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 920 additions and 773 deletions

File diff suppressed because it is too large Load diff

View file

@ -226,6 +226,7 @@ CREATE TABLE `tabDocType` (
`email_append_to` int(1) NOT NULL DEFAULT 0,
`subject_field` varchar(255) DEFAULT NULL,
`sender_field` varchar(255) DEFAULT NULL,
`migration_hash` varchar(255) DEFAULT NULL,
PRIMARY KEY (`name`),
KEY `parent` (`parent`)
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

View file

@ -231,6 +231,7 @@ CREATE TABLE "tabDocType" (
"email_append_to" smallint NOT NULL DEFAULT 0,
"subject_field" varchar(255) DEFAULT NULL,
"sender_field" varchar(255) DEFAULT NULL,
"migration_hash" varchar(255) DEFAULT NULL,
PRIMARY KEY ("name")
) ;

View file

@ -158,7 +158,7 @@ def install_app(name, verbose=False, set_as_patched=True):
if name != "frappe":
add_module_defs(name)
sync_for(name, force=True, sync_everything=True, verbose=verbose, reset_permissions=True)
sync_for(name, force=True, reset_permissions=True)
add_to_installed_apps(name)

View file

@ -18,6 +18,7 @@ from frappe.core.doctype.language.language import sync_languages
from frappe.modules.utils import sync_customizations
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
from frappe.search.website_search import build_index_for_all_routes
from frappe.database.schema import add_column
def migrate(verbose=True, skip_failing=False, skip_search_index=False):
@ -26,9 +27,10 @@ def migrate(verbose=True, skip_failing=False, skip_search_index=False):
- run patches
- sync doctypes (schema)
- sync dashboards
- sync jobs
- sync fixtures
- sync desktop icons
- sync web pages (from /www)
- sync customizations
- sync languages
- sync web pages (from /www)
- run after migrate hooks
'''
@ -51,6 +53,7 @@ Otherwise, check the server logs and ensure that all the required services are r
os.remove(touched_tables_file)
try:
add_column(doctype="DocType", column_name="migration_hash", fieldtype="Data")
frappe.flags.touched_tables = set()
frappe.flags.in_migrate = True
@ -65,7 +68,7 @@ Otherwise, check the server logs and ensure that all the required services are r
frappe.modules.patch_handler.run_all(skip_failing)
# sync
frappe.model.sync.sync_all(verbose=verbose)
frappe.model.sync.sync_all()
frappe.translate.clear_cache()
sync_jobs()
sync_fixtures()

View file

@ -10,62 +10,67 @@ from frappe.modules.import_file import import_file_by_path
from frappe.modules.patch_handler import block_user
from frappe.utils import update_progress_bar
def sync_all(force=0, verbose=False, reset_permissions=False):
def sync_all(force=0, reset_permissions=False):
block_user(True)
for app in frappe.get_installed_apps():
sync_for(app, force, verbose=verbose, reset_permissions=reset_permissions)
sync_for(app, force, reset_permissions=reset_permissions)
block_user(False)
frappe.clear_cache()
def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_permissions=False):
def sync_for(app_name, force=0, reset_permissions=False):
files = []
if app_name == "frappe":
# these need to go first at time of install
for d in (("core", "docfield"),
("core", "docperm"),
("core", "doctype_action"),
("core", "doctype_link"),
("core", "role"),
("core", "has_role"),
("core", "doctype"),
("core", "user"),
("custom", "custom_field"),
("custom", "property_setter"),
("website", "web_form"),
("website", "web_template"),
("website", "web_form_field"),
("website", "portal_menu_item"),
("data_migration", "data_migration_mapping_detail"),
("data_migration", "data_migration_mapping"),
("data_migration", "data_migration_plan_mapping"),
("data_migration", "data_migration_plan"),
("desk", "number_card"),
("desk", "dashboard_chart"),
("desk", "dashboard"),
("desk", "onboarding_permission"),
("desk", "onboarding_step"),
("desk", "onboarding_step_map"),
("desk", "module_onboarding"),
("desk", "workspace_link"),
("desk", "workspace_chart"),
("desk", "workspace_shortcut"),
("desk", "workspace")):
files.append(os.path.join(frappe.get_app_path("frappe"), d[0],
"doctype", d[1], d[1] + ".json"))
FRAPPE_PATH = frappe.get_app_path("frappe")
for core_module in ["docfield", "docperm", "doctype_action", "doctype_link", "role", "has_role", "doctype"]:
files.append(os.path.join(FRAPPE_PATH, "core", "doctype", core_module, f"{core_module}.json"))
for custom_module in ["custom_field", "property_setter"]:
files.append(os.path.join(FRAPPE_PATH, "custom", "doctype", custom_module, f"{custom_module}.json"))
for website_module in ["web_form", "web_template", "web_form_field", "portal_menu_item"]:
files.append(os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json"))
for data_migration_module in [
"data_migration_mapping_detail",
"data_migration_mapping",
"data_migration_plan_mapping",
"data_migration_plan",
]:
files.append(os.path.join(FRAPPE_PATH, "data_migration", "doctype", data_migration_module, f"{data_migration_module}.json"))
for desk_module in [
"number_card",
"dashboard_chart",
"dashboard",
"onboarding_permission",
"onboarding_step",
"onboarding_step_map",
"module_onboarding",
"workspace_link",
"workspace_chart",
"workspace_shortcut",
"workspace",
]:
files.append(os.path.join(FRAPPE_PATH, "desk", "doctype", desk_module, f"{desk_module}.json"))
for module_name in frappe.local.app_modules.get(app_name) or []:
folder = os.path.dirname(frappe.get_module(app_name + "." + module_name).__file__)
get_doc_files(files, folder)
files = get_doc_files(files=files, start_path=folder)
l = len(files)
if l:
for i, doc_path in enumerate(files):
import_file_by_path(doc_path, force=force, ignore_version=True,
reset_permissions=reset_permissions, for_sync=True)
import_file_by_path(doc_path, force=force, ignore_version=True, reset_permissions=reset_permissions)
frappe.db.commit()
@ -75,17 +80,36 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe
# print each progress bar on new line
print()
def get_doc_files(files, start_path):
"""walk and sync all doctypes and pages"""
# load in sequence - warning for devs
document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format',
'web_page', 'website_theme', 'web_form', 'web_template',
'notification', 'print_style',
'data_migration_mapping', 'data_migration_plan',
'workspace', 'onboarding_step', 'module_onboarding', 'form_tour',
'client_script', 'server_script', 'custom_field', 'property_setter']
files = files or []
# load in sequence - warning for devs
document_types = [
"doctype",
"page",
"report",
"dashboard_chart_source",
"print_format",
"web_page",
"website_theme",
"web_form",
"web_template",
"notification",
"print_style",
"data_migration_mapping",
"data_migration_plan",
"workspace",
"onboarding_step",
"module_onboarding",
"form_tour",
"client_script",
"server_script",
"custom_field",
"property_setter",
]
for doctype in document_types:
doctype_path = os.path.join(start_path, doctype)
if os.path.exists(doctype_path):
@ -95,3 +119,5 @@ def get_doc_files(files, start_path):
if os.path.exists(doc_path):
if not doc_path in files:
files.append(doc_path)
return files

View file

@ -1,31 +1,53 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import frappe, os, json
from frappe.modules import get_module_path, scrub_dt_dn
from frappe.utils import get_datetime_str
import hashlib
import json
import os
import frappe
from frappe.model.base_document import get_controller
from frappe.modules import get_module_path, scrub_dt_dn
from frappe.query_builder import DocType
from frappe.utils import get_datetime_str, now
def caclulate_hash(path: str) -> str:
"""Calculate md5 hash of the file in binary mode
Args:
path (str): Path to the file to be hashed
Returns:
str: The calculated hash
"""
hash_md5 = hashlib.md5()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
ignore_values = {
"Report": ["disabled", "prepared_report", "add_total_row"],
"Print Format": ["disabled"],
"Notification": ["enabled"],
"Print Style": ["disabled"],
"Module Onboarding": ['is_complete'],
"Onboarding Step": ['is_complete', 'is_skipped']
"Module Onboarding": ["is_complete"],
"Onboarding Step": ["is_complete", "is_skipped"],
}
ignore_doctypes = [""]
def import_files(module, dt=None, dn=None, force=False, pre_process=None, reset_permissions=False):
if type(module) is list:
out = []
for m in module:
out.append(import_file(m[0], m[1], m[2], force=force, pre_process=pre_process,
reset_permissions=reset_permissions))
out.append(import_file(m[0], m[1], m[2], force=force, pre_process=pre_process, reset_permissions=reset_permissions))
return out
else:
return import_file(module, dt, dn, force=force, pre_process=pre_process,
reset_permissions=reset_permissions)
return import_file(module, dt, dn, force=force, pre_process=pre_process, reset_permissions=reset_permissions)
def import_file(module, dt, dn, force=False, pre_process=None, reset_permissions=False):
"""Sync a file from txt if modifed, return false if not updated"""
@ -33,77 +55,160 @@ def import_file(module, dt, dn, force=False, pre_process=None, reset_permissions
ret = import_file_by_path(path, force, pre_process=pre_process, reset_permissions=reset_permissions)
return ret
def get_file_path(module, dt, dn):
dt, dn = scrub_dt_dn(dt, dn)
path = os.path.join(get_module_path(module),
os.path.join(dt, dn, dn + ".json"))
path = os.path.join(get_module_path(module), os.path.join(dt, dn, f"{dn}.json"))
return path
def import_file_by_path(path, force=False, data_import=False, pre_process=None, ignore_version=None,
reset_permissions=False, for_sync=False):
def import_file_by_path(path: str,force: bool = False,data_import: bool = False,pre_process = None,ignore_version: bool = None,reset_permissions: bool = False):
"""Import file from the given path
Some conditions decide if a file should be imported or not.
Evaluation takes place in the order they are mentioned below.
- Check if `force` is true. Import the file. If not, move ahead.
- Get `db_modified_timestamp`(value of the modified field in the database for the file).
If the return is `none,` this file doesn't exist in the DB, so Import the file. If not, move ahead.
- Check if there is a hash in DB for that file. If there is, Calculate the Hash of the file to import and compare it with the one in DB if they are not equal.
Import the file. If Hash doesn't exist, move ahead.
- Check if `db_modified_timestamp` is older than the timestamp in the file; if it is, we import the file.
If timestamp comparison happens for doctypes, that means the Hash for it doesn't exist.
So, even if the timestamp is newer on DB (When comparing timestamps), we import the file and add the calculated Hash to the DB.
So in the subsequent imports, we can use hashes to compare. As a precautionary measure, the timestamp is updated to the current time as well.
Args:
path (str): Path to the file.
force (bool, optional): Load the file without checking any conditions. Defaults to False.
data_import (bool, optional): [description]. Defaults to False.
pre_process ([type], optional): Any preprocesing that may need to take place on the doc. Defaults to None.
ignore_version (bool, optional): ignore current version. Defaults to None.
reset_permissions (bool, optional): reset permissions for the file. Defaults to False.
Returns:
[bool]: True if import takes place. False if it wasn't imported.
"""
frappe.flags.dt = frappe.flags.dt or []
try:
docs = read_doc_from_file(path)
except IOError:
print (path + " missing")
print(f"{path} missing")
return
calculated_hash = caclulate_hash(path)
if docs:
if not isinstance(docs, list):
docs = [docs]
for doc in docs:
if not force and not is_changed(doc):
return False
original_modified = doc.get("modified")
# modified timestamp in db, none if doctype's first import
db_modified_timestamp = frappe.db.get_value(doc["doctype"], doc["name"], "modified")
is_db_timestamp_latest = db_modified_timestamp and doc.get("modified") <= get_datetime_str(db_modified_timestamp)
import_doc(doc, force=force, data_import=data_import, pre_process=pre_process,
ignore_version=ignore_version, reset_permissions=reset_permissions, path=path)
if not force or db_modified_timestamp:
try:
stored_hash = frappe.db.get_value(doc["doctype"], doc["name"], "migration_hash")
except Exception:
frappe.flags.dt += [doc["doctype"]]
stored_hash = None
if original_modified:
update_modified(original_modified, doc)
# if hash exists and is equal no need to update
if stored_hash and stored_hash == calculated_hash:
return False
# if hash doesn't exist, check if db timestamp is same as json timestamp, add hash if from doctype
if is_db_timestamp_latest and doc["doctype"] != "DocType":
return False
import_doc(
docdict=doc,
force=force,
data_import=data_import,
pre_process=pre_process,
ignore_version=ignore_version,
reset_permissions=reset_permissions,
path=path,
)
if doc["doctype"] == "DocType":
doctype_table = DocType("DocType")
frappe.qb.update(
doctype_table
).set(
doctype_table.migration_hash, calculated_hash
).where(
doctype_table.name == doc["name"]
).run()
new_modified_timestamp = doc.get("modified")
# if db timestamp is newer, hash must have changed, must update db timestamp
if is_db_timestamp_latest and doc["doctype"] == "DocType":
new_modified_timestamp = now()
if new_modified_timestamp:
update_modified(new_modified_timestamp, doc)
return True
def is_changed(doc):
def is_timestamp_changed(doc):
# check if timestamps match
db_modified = frappe.db.get_value(doc['doctype'], doc['name'], 'modified')
if db_modified and doc.get('modified')==get_datetime_str(db_modified):
return False
return True
db_modified = frappe.db.get_value(doc["doctype"], doc["name"], "modified")
return not (db_modified and doc.get("modified") == get_datetime_str(db_modified))
def read_doc_from_file(path):
doc = None
if os.path.exists(path):
with open(path, 'r') as f:
with open(path, "r") as f:
try:
doc = json.loads(f.read())
except ValueError:
print("bad json: {0}".format(path))
raise
else:
raise IOError('%s missing' % path)
raise IOError("%s missing" % path)
return doc
def update_modified(original_modified, doc):
# since there is a new timestamp on the file, update timestamp in
if doc["doctype"] == doc["name"] and doc["name"]!="DocType":
frappe.db.sql("""update tabSingles set value=%s where field="modified" and doctype=%s""",
(original_modified, doc["name"]))
else:
frappe.db.sql("update `tab%s` set modified=%s where name=%s" % (doc['doctype'],
'%s', '%s'), (original_modified, doc['name']))
if doc["doctype"] == doc["name"] and doc["name"] != "DocType":
singles_table = DocType("Singles")
def import_doc(docdict, force=False, data_import=False, pre_process=None,
ignore_version=None, reset_permissions=False, path=None):
frappe.qb.update(
singles_table
).set(
singles_table.value,original_modified
).where(
singles_table.field == "modified"
).where(
singles_table.doctype == doc["name"]
).run()
else:
doctype_table = DocType(doc['doctype'])
frappe.qb.update(doctype_table
).set(
doctype_table.modified, original_modified
).where(
doctype_table.name == doc["name"]
).run()
def import_doc(docdict, force=False, data_import=False, pre_process=None, ignore_version=None, reset_permissions=False, path=None):
frappe.flags.in_import = True
docdict["__islocal"] = 1
controller = get_controller(docdict['doctype'])
if controller and hasattr(controller, 'prepare_for_import') and callable(getattr(controller, 'prepare_for_import')):
controller = get_controller(docdict["doctype"])
if controller and hasattr(controller, "prepare_for_import") and callable(getattr(controller, "prepare_for_import")):
controller.prepare_for_import(docdict)
doc = frappe.get_doc(docdict)
@ -132,15 +237,16 @@ def import_doc(docdict, force=False, data_import=False, pre_process=None,
return doc
def load_code_properties(doc, path):
'''Load code files stored in separate files with extensions'''
"""Load code files stored in separate files with extensions"""
if path:
if hasattr(doc, 'get_code_fields'):
if hasattr(doc, "get_code_fields"):
dirname, filename = os.path.split(path)
for key, extn in doc.get_code_fields().items():
codefile = os.path.join(dirname, filename.split('.')[0]+'.'+extn)
codefile = os.path.join(dirname, filename.split(".")[0] + "." + extn)
if os.path.exists(codefile):
with open(codefile,'r') as txtfile:
with open(codefile, "r") as txtfile:
doc.set(key, txtfile.read())
@ -164,12 +270,13 @@ def delete_old_doc(doc, reset_permissions):
doc.flags.ignore_children_type = ignore
def reset_tree_properties(doc):
# Note on Tree DocTypes:
# The tree structure is maintained in the database via the fields "lft" and
# "rgt". They are automatically set and kept up-to-date. Importing them
# would destroy any existing tree structure.
if getattr(doc.meta, 'is_tree', None) and any([doc.lft, doc.rgt]):
if getattr(doc.meta, "is_tree", None) and any([doc.lft, doc.rgt]):
print('Ignoring values of `lft` and `rgt` for {} "{}"'.format(doc.doctype, doc.name))
doc.lft = None
doc.rgt = None

View file

@ -1,7 +1,7 @@
import frappe
def execute():
frappe.flags.in_patch = True
frappe.reload_doc('core', 'doctype', 'user_permission')
frappe.reload_doc("core", "doctype", "user_permission")
frappe.db.commit()

View file

@ -1,2 +1,2 @@
from pypika import *
from frappe.query_builder.utils import Column, get_query_builder, patch_query_execute
from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute

View file

@ -44,6 +44,9 @@ def get_attr(method_string):
methodname = method_string.split('.')[-1]
return getattr(import_module(modulename), methodname)
def DocType(*args, **kwargs):
return frappe.qb.DocType(*args, **kwargs)
def patch_query_execute():
"""Patch the Query Builder with helper execute method
This excludes the use of `frappe.db.sql` method while