From 421a0353756dd279208036d8ac5bf11226015862 Mon Sep 17 00:00:00 2001 From: Pratik Vyas Date: Mon, 28 Jul 2014 16:46:54 +0530 Subject: [PATCH 1/6] fix get_file_url --- frappe/utils/file_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index c78af726de..2e991ba601 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -15,7 +15,7 @@ class MaxFileSizeReachedError(frappe.ValidationError): pass def get_file_url(file_data_name): data = frappe.db.get_value("File Data", file_data_name, ["file_name", "file_url"], as_dict=True) - return data.file_name or data.file_url + return data.file_url or data.file_name def upload(): # get record details From 4ae03c1cb0235ff264e852cd3f8ae2059cf2a5fd Mon Sep 17 00:00:00 2001 From: Pratik Vyas Date: Mon, 28 Jul 2014 16:58:02 +0530 Subject: [PATCH 2/6] add patch to fix file data --- frappe/patches.txt | 1 + frappe/patches/v4_1/file_manager_fix.py | 31 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 frappe/patches/v4_1/file_manager_fix.py diff --git a/frappe/patches.txt b/frappe/patches.txt index a678367b90..a23f61b899 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -48,3 +48,4 @@ execute:frappe.db.sql("""update `tabSingles` set `value`=`doctype` where `field` frappe.patches.v4_1.enable_print_as_pdf #2014-06-17 execute:frappe.db.sql("""update `tabDocPerm` set email=1 where parent='User' and permlevel=0 and `role`='All' and `read`=1 and apply_user_permissions=1""") #2014-07-15 execute:frappe.db.sql("""update `tabPrint Format` set print_format_type='Client' where ifnull(print_format_type, '')=''""") #2014-07-28 +frappe.patches.v4_1.file_manager_fix diff --git a/frappe/patches/v4_1/file_manager_fix.py b/frappe/patches/v4_1/file_manager_fix.py new file mode 100644 index 0000000000..8831af94bb --- /dev/null +++ b/frappe/patches/v4_1/file_manager_fix.py @@ -0,0 +1,31 @@ +# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals + +import frappe +import os +from frappe.utils import get_files_path +from frappe.utils.file_manager import get_content_hash, get_file + + +def execute(): + for name, file_name, file_url in frappe.db.sql( + """select name, file_name, file_url from `tabFile Data` + where file_name is not null and content_hash is null"""): + b = frappe.get_doc('File Data', name) + old_file_name = b.file_name + b.file_name = os.path.basename(old_file_name) + if old_file_name.startswith('files/') or old_file_name.startswith('/files/'): + b.file_url = os.path.normpath('/' + old_file_name) + else: + b.file_url = os.path.normpath('/files/' + old_file_name) + try: + _file_name, content = get_file(name) + b.content_hash = get_content_hash(content) + except IOError: + print 'Warning: Error processing ', name + _file_name = old_file_name + b.content_hash = None + + b.save() From 683796e0261e6a2d1b0edd7917b869f9ecda589d Mon Sep 17 00:00:00 2001 From: Pratik Vyas Date: Mon, 28 Jul 2014 20:08:21 +0530 Subject: [PATCH 3/6] [minor] use ifnull in filemanager fix patch --- frappe/patches/v4_1/file_manager_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/patches/v4_1/file_manager_fix.py b/frappe/patches/v4_1/file_manager_fix.py index 8831af94bb..060e5503aa 100644 --- a/frappe/patches/v4_1/file_manager_fix.py +++ b/frappe/patches/v4_1/file_manager_fix.py @@ -12,7 +12,7 @@ from frappe.utils.file_manager import get_content_hash, get_file def execute(): for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile Data` - where file_name is not null and content_hash is null"""): + where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""): b = frappe.get_doc('File Data', name) old_file_name = b.file_name b.file_name = os.path.basename(old_file_name) From 3b0cad95029d8d1a3d746c766af17c89710f1706 Mon Sep 17 00:00:00 2001 From: Pratik Vyas Date: Tue, 29 Jul 2014 14:03:57 +0530 Subject: [PATCH 4/6] renaming of replaced files --- frappe/patches.txt | 1 - frappe/patches/v4_1/file_manager_fix.py | 65 +++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/frappe/patches.txt b/frappe/patches.txt index a23f61b899..a678367b90 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -48,4 +48,3 @@ execute:frappe.db.sql("""update `tabSingles` set `value`=`doctype` where `field` frappe.patches.v4_1.enable_print_as_pdf #2014-06-17 execute:frappe.db.sql("""update `tabDocPerm` set email=1 where parent='User' and permlevel=0 and `role`='All' and `read`=1 and apply_user_permissions=1""") #2014-07-15 execute:frappe.db.sql("""update `tabPrint Format` set print_format_type='Client' where ifnull(print_format_type, '')=''""") #2014-07-28 -frappe.patches.v4_1.file_manager_fix diff --git a/frappe/patches/v4_1/file_manager_fix.py b/frappe/patches/v4_1/file_manager_fix.py index 060e5503aa..9221959202 100644 --- a/frappe/patches/v4_1/file_manager_fix.py +++ b/frappe/patches/v4_1/file_manager_fix.py @@ -5,11 +5,23 @@ from __future__ import unicode_literals import frappe import os -from frappe.utils import get_files_path -from frappe.utils.file_manager import get_content_hash, get_file +from frappe.utils.file_manager import get_content_hash, get_file, get_file_name +from frappe.utils import get_files_path, get_site_path +# The files missed by the previous patch might have been replaced with new files +# with the same filename +# +# This patch does the following, +# * Detect which files were replaced and rename them with name{hash:5}.extn and +# update filedata record for the new file +# +# * make missing_files.txt in site dir with files that should be recovered from +# a backup from a time before version 3 migration +# +# * Patch remaining unpatched file data records. def execute(): + rename_replacing_files() for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile Data` where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""): @@ -25,7 +37,54 @@ def execute(): b.content_hash = get_content_hash(content) except IOError: print 'Warning: Error processing ', name - _file_name = old_file_name b.content_hash = None b.save() + +def get_replaced_files(): + ret = [] + new_files = dict(frappe.db.sql("select name, file_name from `tabFile Data` where file_name not like 'files/%'")) + old_files = dict(frappe.db.sql("select name, file_name from `tabFile Data` where ifnull(content_hash, '')=''")) + files = dict(frappe.db.sql("select name, file_name from `tabFile Data` where file_name is not null")) + invfiles = invert_dict(new_files) + + for nname, nfilename in new_files.iteritems(): + if 'files/' + nfilename in old_files.values(): + ret.append((nfilename, invfiles[nfilename])) + return ret + +def rename_replacing_files(): + replaced_files = get_replaced_files() + if len(replaced_files): + missing_files = [v[0] for v in replaced_files] + with open(get_site_path('missing_files.txt'), 'w') as f: + f.write('\n'.join(missing_files) + '\n') + + for file_name, file_datas in replaced_files: + print 'processing ' + file_name + content_hash = frappe.db.get_value('File Data', file_datas[0], 'content_hash') + if not content_hash: + continue + new_file_name = get_file_name(file_name, content_hash) + if os.path.exists(get_files_path(new_file_name)): + continue + print 'skipping ' + file_name + os.rename(get_files_path(file_name), get_files_path(new_file_name)) + for name in file_datas: + f = frappe.get_doc('File Data', name) + f.file_name = new_file_name + f.file_url = '/files/' + new_file_name + f.save() + +def invert_dict(ddict): + ret = {} + for k,v in ddict.iteritems(): + if not ret.get(v): + ret[v] = [k] + else: + ret[v].append(k) + return ret + +def get_file_name(fname, hash): + partial, extn = fname.rsplit('.', 1) + return '{partial}{suffix}.{extn}'.format(partial=partial, extn=extn, suffix=hash[:5]) From c98b84079d8e5df596a1c56e3e917590d0dd9f47 Mon Sep 17 00:00:00 2001 From: Pratik Vyas Date: Tue, 29 Jul 2014 14:57:24 +0530 Subject: [PATCH 5/6] [fix] [patch] remove dead query --- frappe/patches/v4_1/file_manager_fix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/patches/v4_1/file_manager_fix.py b/frappe/patches/v4_1/file_manager_fix.py index 9221959202..c0dd1fecee 100644 --- a/frappe/patches/v4_1/file_manager_fix.py +++ b/frappe/patches/v4_1/file_manager_fix.py @@ -45,7 +45,6 @@ def get_replaced_files(): ret = [] new_files = dict(frappe.db.sql("select name, file_name from `tabFile Data` where file_name not like 'files/%'")) old_files = dict(frappe.db.sql("select name, file_name from `tabFile Data` where ifnull(content_hash, '')=''")) - files = dict(frappe.db.sql("select name, file_name from `tabFile Data` where file_name is not null")) invfiles = invert_dict(new_files) for nname, nfilename in new_files.iteritems(): From 798c1a1fa78e7f3a656a00f83fe2715140e89499 Mon Sep 17 00:00:00 2001 From: Pratik Vyas Date: Tue, 29 Jul 2014 15:56:40 +0530 Subject: [PATCH 6/6] [minor] add patch to patches.txt --- frappe/patches.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/patches.txt b/frappe/patches.txt index a678367b90..a23f61b899 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -48,3 +48,4 @@ execute:frappe.db.sql("""update `tabSingles` set `value`=`doctype` where `field` frappe.patches.v4_1.enable_print_as_pdf #2014-06-17 execute:frappe.db.sql("""update `tabDocPerm` set email=1 where parent='User' and permlevel=0 and `role`='All' and `read`=1 and apply_user_permissions=1""") #2014-07-15 execute:frappe.db.sql("""update `tabPrint Format` set print_format_type='Client' where ifnull(print_format_type, '')=''""") #2014-07-28 +frappe.patches.v4_1.file_manager_fix