From 22ba310aaf4ef16f96552b4e7874b36fb48b85fb Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Mon, 3 Sep 2018 18:04:45 +0530 Subject: [PATCH] [2/3] file-api: code migration migrate api from file_manager.py to file.py Signed-off-by: Chinmay Pai --- frappe/core/doctype/communication/email.py | 2 +- frappe/core/doctype/data_import/importer.py | 2 +- frappe/core/doctype/file/file.py | 200 +++++++++--------- frappe/core/doctype/file/test_file.py | 2 +- .../prepared_report/prepared_report.py | 4 +- frappe/desk/form/utils.py | 5 +- .../email_account/test_email_account.py | 7 +- frappe/email/email_body.py | 2 +- frappe/email/queue.py | 2 +- frappe/email/receive.py | 2 +- frappe/handler.py | 1 - frappe/limits.py | 2 +- frappe/model/delete_doc.py | 2 +- frappe/patches/v4_0/file_manager_hooks.py | 2 +- frappe/patches/v4_1/file_manager_fix.py | 2 +- frappe/public/js/frappe/upload.js | 2 +- frappe/tests/test_filemanager.py | 2 +- frappe/utils/csvutils.py | 2 +- frappe/utils/xlsxutils.py | 2 +- frappe/website/doctype/web_form/web_form.py | 10 +- 20 files changed, 129 insertions(+), 126 deletions(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 00f1dc9abf..6303eb7fe3 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -10,7 +10,7 @@ from email.utils import formataddr from frappe.core.utils import get_parent_doc from frappe.utils import (get_url, get_formatted_email, cint, validate_email_add, split_emails, time_diff_in_seconds, parse_addr, get_datetime) -from frappe.utils.file_manager import get_file +from frappe.core.doctype.file.file import get_file from frappe.email.queue import check_email_limit from frappe.utils.scheduler import log from frappe.email.email_body import get_message_id diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index ac00851ebf..69a1fa3190 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -272,7 +272,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, # header filename, file_extension = ['',''] if not rows: - from frappe.utils.file_manager import get_file # get_file_doc + from frappe.core.doctype.file.file import get_file # get_file_doc fname, fcontent = get_file(data_import_doc.import_file) filename, file_extension = os.path.splitext(fname) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index b3624bed60..c9cdb25c55 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -199,7 +199,7 @@ class File(NestedSet): self.check_folder_is_empty() self.check_reference_doc_permission() super(File, self).on_trash() - self.delete_file() + self.call_delete_file() def make_thumbnail(self, set_as_thumbnail=True, width=300, height=300, suffix="small", crop=False): if self.file_url: @@ -265,14 +265,14 @@ class File(NestedSet): except frappe.DoesNotExistError: pass - def delete_file(self): + def call_delete_file(self): """If file not attached to any other record, delete it""" if self.file_name and self.content_hash and (not frappe.db.count("File", {"content_hash": self.content_hash, "name": ["!=", self.name]})): - delete_file_data_content(self) + self.delete_file_data_content() elif self.file_url: - delete_file_data_content(self, only_thumbnail=True) + self.delete_file_data_content(only_thumbnail=True) def on_rollback(self): self.flags.on_rollback = True @@ -308,17 +308,6 @@ class File(NestedSet): frappe.delete_doc('File', self.name) - def remove_file_by_url(self): - if self.attached_to_doctype and self.attached_to_name: - fid = frappe.db.get_value("File", {"file_url": self.file_url, - "attached_to_doctype": self.attached_to_doctype, "attached_to_name": self.attached_to_name}) - else: - fid = frappe.db.get_value("File", {"file_url": self.file_url}) - - if fid: - return remove_file(fid) - - def get_file_url(self): data = frappe.db.get_value("File", self.file_data_name, ["file_name", "file_url"], as_dict=True) return data.file_url or data.file_name @@ -434,10 +423,10 @@ class File(NestedSet): self.content = self.content.split(b",")[1] self.content = base64.b64decode(self.content) - file_size = check_max_file_size(self.content) + file_size = self.check_max_file_size() self.content_hash = get_content_hash(self.content) self.content_type = mimetypes.guess_type(self.fname)[0] - self.fname = self.get_file_name(content_hash[-6:]) + self.fname = get_file_name(self.fname, self.content_hash[-6:]) self.is_private = is_private file_data = self.get_file_data_from_hash(is_private=self.is_private) if not file_data: @@ -477,7 +466,7 @@ class File(NestedSet): def save_file_on_filesystem(self): - fpath = write_file(self.content, self.fname, self.is_private) + fpath = self.write_file(self, is_private=self.is_private) if self.is_private: file_url = "/private/files/{0}".format(self.fname) @@ -490,6 +479,89 @@ class File(NestedSet): } + def check_max_file_size(self): + max_file_size = get_max_file_size() + file_size = len(self.content) + + if file_size > max_file_size: + frappe.msgprint(_("File size exceeded the maximum allowed size of {0} MB").format( + max_file_size / 1048576), + raise_exception=MaxFileSizeReachedError) + + return file_size + + + def write_file(self, is_private=0): + """write file to disk with a random name (to compare)""" + file_path = get_files_path(is_private=is_private) + + # create directory (if not exists) + frappe.create_folder(file_path) + # write the file + if isinstance(self.content, text_type): + self.content = self.content.encode() + with open(os.path.join(file_path.encode('utf-8'), self.fname.encode('utf-8')), 'wb+') as f: + f.write(self.content) + + return get_files_path(self.fname, is_private=is_private) + + + def remove_file(self, attached_to_doctype=None, attached_to_name=None, from_delete=False): + """Remove file and File entry""" + file_name = None + if not (attached_to_doctype and attached_to_name): + attached = frappe.db.get_value("File", fid, + ["attached_to_doctype", "attached_to_name", "file_name"]) + if attached: + attached_to_doctype, attached_to_name, file_name = attached + + ignore_permissions, comment = False, None + if attached_to_doctype and attached_to_name and not from_delete: + doc = frappe.get_doc(attached_to_doctype, attached_to_name) + ignore_permissions = doc.has_permission("write") or False + if frappe.flags.in_web_form: + ignore_permissions = True + if not file_name: + file_name = frappe.db.get_value("File", fid, "file_name") + comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name)) + + frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions) + + return comment + + + def delete_file_data_content(self, only_thumbnail=False): + method = get_hook_method('delete_file_data_content', fallback=self.delete_file_from_filesystem) + method(self.doc, only_thumbnail=only_thumbnail) + + + def delete_file_from_filesystem(self, only_thumbnail=False): + """Delete file, thumbnail from File document""" + if only_thumbnail: + self.delete_file(self.doc.thumbnail_url) + else: + self.delete_file(self.doc.file_url) + self.delete_file(self.doc.thumbnail_url) + + + def delete_file(self, path): + """Delete file from `public folder`""" + if path: + if ".." in path.split("/"): + frappe.msgprint(_("It is risky to delete this file: {0}. Please contact your System Manager.").format(path)) + + parts = os.path.split(path.strip("/")) + if parts[0]=="files": + path = frappe.utils.get_site_path("public", "files", parts[-1]) + + else: + path = frappe.utils.get_site_path("private", "files", parts[-1]) + + path = encode(path) + if os.path.exists(path): + os.remove(path) + + def on_doctype_update(): frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"]) @@ -622,97 +694,27 @@ def get_max_file_size(): return conf.get('max_file_size') or 10485760 -def check_max_file_size(content): - max_file_size = get_max_file_size() - file_size = len(content) - - if file_size > max_file_size: - frappe.msgprint(_("File size exceeded the maximum allowed size of {0} MB").format( - max_file_size / 1048576), - raise_exception=MaxFileSizeReachedError) - - return file_size - - -def write_file(content, fname, is_private=0): - """write file to disk with a random name (to compare)""" - file_path = get_files_path(is_private=is_private) - - # create directory (if not exists) - frappe.create_folder(file_path) - # write the file - if isinstance(content, text_type): - content = content.encode() - with open(os.path.join(file_path.encode('utf-8'), fname.encode('utf-8')), 'wb+') as f: - f.write(content) - - return get_files_path(fname, is_private=is_private) - - def remove_all(dt, dn, from_delete=False): """remove all files in a transaction""" try: for fid in frappe.db.sql_list("""select name from `tabFile` where attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)): - remove_file(fid, dt, dn, from_delete) + _file = frappe.get_doc("File", {"fid": fid, "attached_to_doctype": dt, "attached_to_name": dn}) + _file.remove_file(from_delete=from_delete) except Exception as e: if e.args[0]!=1054: raise # (temp till for patched) -def remove_file(fid, attached_to_doctype=None, attached_to_name=None, from_delete=False): - """Remove file and File entry""" - file_name = None - if not (attached_to_doctype and attached_to_name): - attached = frappe.db.get_value("File", fid, - ["attached_to_doctype", "attached_to_name", "file_name"]) - if attached: - attached_to_doctype, attached_to_name, file_name = attached - - ignore_permissions, comment = False, None - if attached_to_doctype and attached_to_name and not from_delete: - doc = frappe.get_doc(attached_to_doctype, attached_to_name) - ignore_permissions = doc.has_permission("write") or False - if frappe.flags.in_web_form: - ignore_permissions = True - if not file_name: - file_name = frappe.db.get_value("File", fid, "file_name") - comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name)) - - frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions) - - return comment - - -def delete_file_data_content(doc, only_thumbnail=False): - method = get_hook_method('delete_file_data_content', fallback=delete_file_from_filesystem) - method(doc, only_thumbnail=only_thumbnail) - - -def delete_file_from_filesystem(doc, only_thumbnail=False): - """Delete file, thumbnail from File document""" - if only_thumbnail: - delete_file(doc.thumbnail_url) +def remove_file_by_url(file_url, doctype=None, name=None): + if doctype and name: + fid = frappe.db.get_value("File", {"file_url": file_url, + "attached_to_doctype": doctype, "attached_to_name": name}) else: - delete_file(doc.file_url) - delete_file(doc.thumbnail_url) + fid = frappe.db.get_value("File", {"file_url": file_url}) - -def delete_file(path): - """Delete file from `public folder`""" - if path: - if ".." in path.split("/"): - frappe.msgprint(_("It is risky to delete this file: {0}. Please contact your System Manager.").format(path)) - - parts = os.path.split(path.strip("/")) - if parts[0]=="files": - path = frappe.utils.get_site_path("public", "files", parts[-1]) - - else: - path = frappe.utils.get_site_path("private", "files", parts[-1]) - - path = encode(path) - if os.path.exists(path): - os.remove(path) + if fid: + _file = frappe.get_doc("File", {"fid": fid}) + return _file.remove_file() def get_file(fname): @@ -788,7 +790,7 @@ def download_file(file_url): token is required to download private files. Method : GET - Endpoint : frappe.utils.file_manager.download_file + Endpoint : frappe.core.doctype.file.file.download_file URL Params : file_name = /path/to/file relative to site path """ file_doc = frappe.get_doc("File", {"file_url":file_url}) diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index fbf61fbff5..be7758e439 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -101,7 +101,7 @@ class TestFile(unittest.TestCase): self.assertRaises(frappe.ValidationError, folder.delete) def test_file_upload_limit(self): - from frappe.utils.file_manager import MaxFileSizeReachedError + from frappe.core.doctype.file.file import MaxFileSizeReachedError from frappe.limits import update_limits, clear_limit from frappe import _dict diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py index 9565a18092..9574320ee1 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.py +++ b/frappe/core/doctype/prepared_report/prepared_report.py @@ -11,10 +11,10 @@ import frappe from frappe.model.document import Document from frappe.utils.background_jobs import enqueue from frappe.desk.query_report import generate_report_result, get_columns_dict -from frappe.utils.file_manager import remove_all +from frappe.core.doctype.file.file import remove_all from frappe.utils.csvutils import to_csv, read_csv_content_from_attached_file from frappe.desk.form.load import get_attachments -from frappe.utils.file_manager import download_file +from frappe.core.doctype.file.file import download_file class PreparedReport(Document): diff --git a/frappe/desk/form/utils.py b/frappe/desk/form/utils.py index bad7657a8e..afb734d7fe 100644 --- a/frappe/desk/form/utils.py +++ b/frappe/desk/form/utils.py @@ -13,9 +13,10 @@ from six import string_types @frappe.whitelist() def remove_attach(): """remove attachment""" - import frappe.utils.file_manager fid = frappe.form_dict.get('fid') - return frappe.utils.file_manager.remove_file(fid) + _file = frappe.get_doc("File", {"fid": fid}) + return _file.remove_file() + @frappe.whitelist() def validate_link(): diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index bae0229a8c..d59c3a62f6 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -8,7 +8,6 @@ test_records = frappe.get_test_records('Email Account') from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments -from frappe.utils.file_manager import delete_file_from_filesystem from frappe.email.doctype.email_account.email_account import notify_unreplied from datetime import datetime, timedelta @@ -55,7 +54,8 @@ class TestEmailAccount(unittest.TestCase): frappe.db.sql("delete from tabCommunication where sender='test_sender@example.com'") existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) - delete_file_from_filesystem(existing_file) + _file = frappe.get_doc("File", {"doc": existing_file}) + _file.delete_file_from_filesystem() with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile: test_mails = [testfile.read()] @@ -73,7 +73,8 @@ class TestEmailAccount(unittest.TestCase): # cleanup existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) - delete_file_from_filesystem(existing_file) + _file = frappe.get_doc("File", {"doc": existing_file}) + def test_incoming_attached_email_from_outlook_plain_text_only(self): frappe.db.sql("delete from tabCommunication where sender='test_sender@example.com'") diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 8ab7ae5c85..e886bc3f06 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -144,7 +144,7 @@ class EMail: def attach_file(self, n): """attach a file from the `FileData` table""" - from frappe.utils.file_manager import get_file + from frappe.core.doctype.file.file import get_file res = get_file(n) if not res: return diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 815ded36b9..5d2acb00c6 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -11,7 +11,7 @@ from frappe.email.email_body import get_email, get_formatted_html, add_attachmen from frappe.utils.verified_command import get_signed_params, verify_request from html2text import html2text from frappe.utils import get_url, nowdate, encode, now_datetime, add_days, split_emails, cstr, cint -from frappe.utils.file_manager import get_file +from frappe.core.doctype.file.file import get_file from rq.timeouts import JobTimeoutException from frappe.utils.scheduler import log from six import text_type, string_types diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 0282eb4bf6..2d50b8c294 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -13,7 +13,7 @@ from frappe import _, safe_decode, safe_encode from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now, cint, cstr, strip, markdown, parse_addr) from frappe.utils.scheduler import log -from frappe.utils.file_manager import get_random_filename, MaxFileSizeReachedError +from frappe.core.doctype.file.file import get_random_filename, MaxFileSizeReachedError class EmailSizeExceededError(frappe.ValidationError): pass class EmailTimeoutError(frappe.ValidationError): pass diff --git a/frappe/handler.py b/frappe/handler.py index 82a19ec558..383b5d0fd0 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -6,7 +6,6 @@ import frappe from frappe import _ import frappe.utils import frappe.sessions -import frappe.utils.file_manager import frappe.desk.form.run_method from frappe.utils.response import build_response from werkzeug.wrappers import Response diff --git a/frappe/limits.py b/frappe/limits.py index 0330db2567..55dd42a09a 100755 --- a/frappe/limits.py +++ b/frappe/limits.py @@ -173,7 +173,7 @@ def clear_limit(key): def validate_space_limit(file_size): """Stop from writing file if max space limit is reached""" - from frappe.utils.file_manager import MaxFileSizeReachedError + from frappe.core.doctype.file.file import MaxFileSizeReachedError limits = get_limits() if not limits.space: diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 6348f66b55..b16c74abbd 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -7,7 +7,7 @@ import frappe import frappe.model.meta from frappe.model.dynamic_links import get_dynamic_link_map import frappe.defaults -from frappe.utils.file_manager import remove_all +from frappe.core.doctype.file.file import remove_all from frappe.utils.password import delete_all_passwords_for from frappe import _ from frappe.model.naming import revert_series_if_last diff --git a/frappe/patches/v4_0/file_manager_hooks.py b/frappe/patches/v4_0/file_manager_hooks.py index 60686b1e0e..018e52ca75 100644 --- a/frappe/patches/v4_0/file_manager_hooks.py +++ b/frappe/patches/v4_0/file_manager_hooks.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals, print_function import frappe import os from frappe.utils import get_files_path -from frappe.utils.file_manager import get_content_hash, get_file +from frappe.core.doctype.file.file import get_content_hash, get_file def execute(): diff --git a/frappe/patches/v4_1/file_manager_fix.py b/frappe/patches/v4_1/file_manager_fix.py index 1fced61799..edb450d21a 100644 --- a/frappe/patches/v4_1/file_manager_fix.py +++ b/frappe/patches/v4_1/file_manager_fix.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals, print_function import frappe import os -from frappe.utils.file_manager import get_content_hash, get_file, get_file_name +from frappe.core.doctype.file.file import get_content_hash, get_file, get_file_name from frappe.utils import get_files_path, get_site_path # The files missed by the previous patch might have been replaced with new files diff --git a/frappe/public/js/frappe/upload.js b/frappe/public/js/frappe/upload.js index bdc8b41316..ca2ed264ed 100644 --- a/frappe/public/js/frappe/upload.js +++ b/frappe/public/js/frappe/upload.js @@ -317,7 +317,7 @@ frappe.upload = { } else { args.file_size = fileobj.size; frappe.call({ - method: 'frappe.utils.file_manager.validate_filename', + method: 'frappe.core.doctype.file.file.validate_filename', args: {"filename": args.filename}, callback: function(r) { args.filename = r.message; diff --git a/frappe/tests/test_filemanager.py b/frappe/tests/test_filemanager.py index a25c5a7503..4997e399cc 100644 --- a/frappe/tests/test_filemanager.py +++ b/frappe/tests/test_filemanager.py @@ -6,7 +6,7 @@ import frappe import os import unittest -from frappe.utils.file_manager import get_file, get_files_path +from frappe.core.doctype.file.file import get_file, get_files_path test_content1 = 'Hello' test_content2 = 'Hello World' diff --git a/frappe/utils/csvutils.py b/frappe/utils/csvutils.py index b9159a4bee..db66087d73 100644 --- a/frappe/utils/csvutils.py +++ b/frappe/utils/csvutils.py @@ -30,7 +30,7 @@ def read_csv_content_from_attached_file(doc): raise Exception try: - from frappe.utils.file_manager import get_file + from frappe.core.doctype.file.file import get_file fname, fcontent = get_file(fileid) return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors')) except Exception: diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 880726c16e..b7c26ba5c3 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -70,7 +70,7 @@ def handle_html(data): def read_xlsx_file_from_attached_file(file_id=None, fcontent=None, filepath=None): if file_id: - from frappe.utils.file_manager import get_file_path + from frappe.core.doctype.file.file import get_file_path filename = get_file_path(file_id) elif fcontent: from io import BytesIO diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index d8d2418c45..aa95198d3d 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -8,7 +8,8 @@ from frappe import _, scrub from frappe.utils import cstr from frappe.website.utils import get_comment_list from frappe.custom.doctype.customize_form.customize_form import docfield_properties -from frappe.utils.file_manager import get_max_file_size +from frappe.core.doctype.file.file import get_max_file_size +from frappe.core.doctype.file.file import remove_file_by_url from frappe.modules.utils import export_module_json, get_doc_module from six.moves.urllib.parse import urlencode from frappe.integrations.utils import get_payment_gateway_controller @@ -415,8 +416,7 @@ def accept(web_form, data, for_payment=False): # remove earlier attached file (if exists) if doc.get(fieldname): - file = frappe.get_doc("File", {"attached_to_doctype": doc.doctype, "file_url": doc.get(fieldname), "attached_to_name": doc.name}) - file.remove_file_by_url() + remove_file_by_url(doc.get(fieldname), doctype=doc.doctype, name=doc.name) # save new file filename, dataurl = filedata.split(',', 1) @@ -432,8 +432,8 @@ def accept(web_form, data, for_payment=False): if files_to_delete: for f in files_to_delete: if f: - file = frappe.get_doc("File", {"attached_to_doctype": doc.doctype, "file_url": doc.get(fieldname), "attached_to_name": doc.name}) - file.remove_file_by_url() + remove_file_by_url(doc.get(fieldname), doctype=doc.doctype, name=doc.name) + frappe.flags.web_form_doc = doc