diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index d859041e4c..9434def556 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -10,7 +10,6 @@ from email.utils import formataddr from frappe.core.utils import get_parent_doc from frappe.utils import (get_url, get_formatted_email, cint, validate_email_add, split_emails, time_diff_in_seconds, parse_addr, get_datetime) -from frappe.core.doctype.file.file import get_file from frappe.email.queue import check_email_limit from frappe.utils.scheduler import log from frappe.email.email_body import get_message_id @@ -285,7 +284,8 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None) # is it a filename? try: # keep this for error handling - file = get_file(a) + _file = frappe.get_doc("File", {"file_name": a}) + content = _file.get_content() # these attachments will be attached on-demand # and won't be stored in the message doc.attachments.append({"fid": a}) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 36775e7d78..b281818f8a 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -272,9 +272,9 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, # header filename, file_extension = ['',''] if not rows: - from frappe.core.doctype.file.file import get_file # get_file_doc - fname, fcontent = get_file(data_import_doc.import_file) - filename, file_extension = os.path.splitext(fname) + _file = frappe.get_doc("File", {"file_name": data_import_doc.import_file}) + fcontent = _file.get_content() + filename, file_extension = os.path.splitext(_file.file_name) if file_extension == '.xlsx' and from_data_import == 'Yes': from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index d3a50990a1..07e0e97ff3 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -345,6 +345,61 @@ class File(NestedSet): "comment": comment.as_dict() if comment else {} } + def get_content(self): + """Returns [`file_name`, `content`] for given file name `fname`""" + if self.get('content'): + return self.content + file_path = self.get_full_path() + + # read the file + if PY2: + with open(encode(file_path)) as f: + content = f.read() + else: + with io.open(encode(file_path), mode='rb') as f: + content = f.read() + try: + # for plain text files + content = content.decode() + except UnicodeDecodeError: + # for .png, .jpg, etc + pass + + return content + + def get_full_path(self): + """Returns file path from given file name""" + + file_path = self.file_url + + if "/" not in file_path: + file_path = "/files/" + file_path + + if file_path.startswith("/private/files/"): + file_path = get_files_path(*file_path.split("/private/files/", 1)[1].split("/"), is_private=1) + + elif file_path.startswith("/files/"): + file_path = get_files_path(*file_path.split("/files/", 1)[1].split("/")) + + else: + frappe.throw(_("There is some problem with the file url: {0}").format(file_path)) + + return file_path + + def write_file(self): + """write file to disk with a random name (to compare)""" + file_path = get_files_path(is_private=self.is_private) + + # create directory (if not exists) + frappe.create_folder(file_path) + # write the file + self.content = self.get_content() + if isinstance(self.content, text_type): + self.content = self.content.encode() + with open(os.path.join(file_path.encode('utf-8'), self.file_name.encode('utf-8')), 'wb+') as f: + f.write(self.content) + + return get_files_path(self.file_name, is_private=self.is_private) def get_file_doc(self): '''returns File object (Document) from given parameters or form_dict''' @@ -368,7 +423,7 @@ class File(NestedSet): def save_uploaded(self): - self.file_name, self.content = self.get_uploaded_content() + self.content = self.get_uploaded_content() if self.content: return self.save_file(content=self.content) else: @@ -408,12 +463,11 @@ class File(NestedSet): if "," in frappe.form_dict.filedata: frappe.form_dict.filedata = frappe.form_dict.filedata.rsplit(",", 1)[1] frappe.uploaded_content = base64.b64decode(frappe.form_dict.filedata) - frappe.uploaded_filename = frappe.form_dict.filename - return frappe.uploaded_filename, frappe.uploaded_content - elif self.content and self.file_name: - return self.file_name, self.content + return frappe.uploaded_content + elif self.content: + return self.content frappe.msgprint(_('No file attached')) - return None, None + return None def save_file(self, content=None, decode=False): @@ -436,8 +490,11 @@ class File(NestedSet): if not file_data: call_hook_method("before_write_file", file_size=self.file_size) - write_file_method = get_hook_method('write_file', fallback=save_file_on_filesystem) - file_data = write_file_method(self.file_name, self.content, self.content_type, self.is_private) + write_file_method = get_hook_method('write_file') + if write_file_method: + file_data = write_file_method(self) + else: + file_data = self.save_file_on_filesystem() file_data = copy(file_data) file_data.update({ @@ -460,6 +517,18 @@ class File(NestedSet): return f + def save_file_on_filesystem(self): + fpath = self.write_file() + + if self.is_private: + self.file_url = "/private/files/{0}".format(self.file_name) + else: + self.file_url = "/files/{0}".format(self.file_name) + + return { + 'file_name': os.path.basename(fpath), + 'file_url': self.file_url + } def get_file_data_from_hash(self): for name in frappe.db.sql_list("select name from `tabFile` where content_hash=%s and is_private=%s", @@ -482,8 +551,31 @@ class File(NestedSet): def delete_file_data_content(self, only_thumbnail=False): - method = get_hook_method('delete_file_data_content', fallback=delete_file_from_filesystem) - method(self, only_thumbnail=only_thumbnail) + method = get_hook_method('delete_file_data_content') + if method: + method(self, only_thumbnail=only_thumbnail) + else: + self.delete_file_from_filesystem(only_thumbnail=only_thumbnail) + + + def delete_file_from_filesystem(self, only_thumbnail=False): + """Delete file, thumbnail from File document""" + if only_thumbnail: + delete_file(self.thumbnail_url) + else: + delete_file(self.file_url) + delete_file(self.thumbnail_url) + + + def check_file_permission(self): + for file in frappe.get_all("File", filters={"file_url": self.file_url, "is_private": 1}, + fields=["name", "attached_to_doctype", "attached_to_name"]): + + if (frappe.has_permission("File", ptype="read", doc=file.name) + or frappe.has_permission(file.attached_to_doctype, ptype="read", doc=file.attached_to_name)): + return True + + raise frappe.PermissionError def on_doctype_update(): @@ -615,44 +707,6 @@ def get_web_image(file_url): return image, filename, extn -def save_file_on_filesystem(file_name, content, content_type, is_private): - fpath = write_file(file_name, content, is_private=is_private) - - if is_private: - file_url = "/private/files/{0}".format(file_name) - else: - file_url = "/files/{0}".format(file_name) - - return { - 'file_name': os.path.basename(fpath), - 'file_url': file_url - } - - -def write_file(file_name, content, is_private=0): - """write file to disk with a random name (to compare)""" - file_path = get_files_path(is_private=is_private) - - # create directory (if not exists) - frappe.create_folder(file_path) - # write the file - if isinstance(content, text_type): - content = content.encode() - with open(os.path.join(file_path.encode('utf-8'), file_name.encode('utf-8')), 'wb+') as f: - f.write(content) - - return get_files_path(file_name, is_private=is_private) - - -def delete_file_from_filesystem(doc, only_thumbnail=False): - """Delete file, thumbnail from File document""" - if only_thumbnail: - delete_file(doc.thumbnail_url) - else: - delete_file(doc.file_url) - delete_file(doc.thumbnail_url) - - def delete_file(path): """Delete file from `public folder`""" if path: @@ -719,51 +773,6 @@ def remove_file_by_url(file_url, doctype=None, name=None): return remove_file(fid=fid) -def get_file(fname): - """Returns [`file_name`, `content`] for given file name `fname`""" - file_path = get_file_path(fname) - - # read the file - if PY2: - with open(encode(file_path)) as f: - content = f.read() - else: - with io.open(encode(file_path), mode='rb') as f: - content = f.read() - try: - # for plain text files - content = content.decode() - except UnicodeDecodeError: - # for .png, .jpg, etc - pass - - return [file_path.rsplit("/", 1)[-1], content] - - -def get_file_path(file_name): - """Returns file path from given file name""" - f = frappe.db.sql("""select file_url from `tabFile` - where name=%s or file_name=%s""", (file_name, file_name)) - if f: - file_name = f[0][0] - - file_path = file_name - - if "/" not in file_path: - file_path = "/files/" + file_path - - if file_path.startswith("/private/files/"): - file_path = get_files_path(*file_path.split("/private/files/", 1)[1].split("/"), is_private=1) - - elif file_path.startswith("/files/"): - file_path = get_files_path(*file_path.split("/files/", 1)[1].split("/")) - - else: - frappe.throw(_("There is some problem with the file url: {0}").format(file_path)) - - return file_path - - def get_content_hash(content): if isinstance(content, text_type): content = content.encode() @@ -795,7 +804,7 @@ def download_file(file_url): Endpoint : frappe.core.doctype.file.file.download_file URL Params : file_name = /path/to/file relative to site path """ - file_doc = frappe.get_doc("File", {"file_url":file_url}) + file_doc = frappe.get_doc("File", {"file_url": file_url}) file_doc.check_permission("read") path = os.path.join(get_files_path(), os.path.basename(file_url)) @@ -859,16 +868,6 @@ def get_random_filename(extn=None, content_type=None): return random_string(7) + (extn or "") -def check_file_permission(file_url): - for file in frappe.get_all("File", filters={"file_url": file_url, "is_private": 1}, fields=["name", "attached_to_doctype", "attached_to_name"]): - - if (frappe.has_permission("File", ptype="read", doc=file.name) - or frappe.has_permission(file.attached_to_doctype, ptype="read", doc=file.attached_to_name)): - return True - - raise frappe.PermissionError - - @frappe.whitelist() def unzip_file(name): '''Unzip the given file and make file records for each of the extracted files''' diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 624f380395..95f2757fb9 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -7,7 +7,7 @@ import frappe import os import unittest from frappe import _ -from frappe.core.doctype.file.file import move_file, get_file, get_files_path +from frappe.core.doctype.file.file import move_file, get_files_path # test_records = frappe.get_test_records('File') test_content1 = 'Hello' @@ -33,7 +33,8 @@ class TestSimpleFile(unittest.TestCase): self.saved_filename = get_files_path(self.saved_file.file_name) def test_save(self): - _, content = get_file(self.saved_file.name) + _file = frappe.get_doc("File", {"file_name": self.saved_file.file_name}) + content = _file.get_content() self.assertEqual(content, self.test_content) def tearDown(self): @@ -61,9 +62,11 @@ class TestSameFileName(unittest.TestCase): self.saved_filename2 = get_files_path(self.saved_file2.file_name) def test_saved_content(self): - _, content1 = get_file(self.saved_file1.name) + _file = frappe.get_doc("File", {"file_name": self.saved_file1.file_name}) + content1 = _file.get_content() self.assertEqual(content1, self.test_content1) - _, content2 = get_file(self.saved_file2.name) + _file = frappe.get_doc("File", {"file_name": self.saved_file2.file_name}) + content2 = _file.get_content() self.assertEqual(content2, self.test_content2) def tearDown(self): @@ -94,8 +97,10 @@ class TestSameContent(unittest.TestCase): self.saved_filename2 = get_files_path(self.saved_file2.file_name) def test_saved_content(self): - filename1, content1 = get_file(self.saved_file1.name) - filename2, content2 = get_file(self.saved_file2.name) + _file1 = frappe.get_doc("File", {"file_name": self.saved_file1.file_name}) + filename1 = _file1.file_name + _file2 = frappe.get_doc("File", {"file_name": self.saved_file2.file_name}) + filename2 = _file2.file_name self.assertEqual(filename1, filename2) self.assertFalse(os.path.exists(get_files_path(self.dup_filename))) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index d7780975ea..bd8a3f72b6 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -8,7 +8,6 @@ test_records = frappe.get_test_records('Email Account') from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments -from frappe.core.doctype.file.file import delete_file_from_filesystem from frappe.email.doctype.email_account.email_account import notify_unreplied from datetime import datetime, timedelta @@ -55,7 +54,7 @@ class TestEmailAccount(unittest.TestCase): frappe.db.sql("delete from tabCommunication where sender='test_sender@example.com'") existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) - delete_file_from_filesystem(existing_file) + existing_file.delete_file_from_filesystem() with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile: test_mails = [testfile.read()] @@ -73,7 +72,7 @@ class TestEmailAccount(unittest.TestCase): # cleanup existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) - delete_file_from_filesystem(existing_file) + existing_file.delete_file_from_filesystem() def test_incoming_attached_email_from_outlook_plain_text_only(self): diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index e886bc3f06..7cb7be1945 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -144,12 +144,12 @@ class EMail: def attach_file(self, n): """attach a file from the `FileData` table""" - from frappe.core.doctype.file.file import get_file - res = get_file(n) - if not res: + _file = frappe.get_doc("File", {"file_name": n}) + content = _file.get_content() + if not content: return - self.add_attachment(res[0], res[1]) + self.add_attachment(_file.file_name, content) def add_attachment(self, fname, fcontent, content_type=None, parent=None, content_id=None, inline=False): diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 5d2acb00c6..8cef5b8164 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -11,7 +11,6 @@ from frappe.email.email_body import get_email, get_formatted_html, add_attachmen from frappe.utils.verified_command import get_signed_params, verify_request from html2text import html2text from frappe.utils import get_url, nowdate, encode, now_datetime, add_days, split_emails, cstr, cint -from frappe.core.doctype.file.file import get_file from rq.timeouts import JobTimeoutException from frappe.utils.scheduler import log from six import text_type, string_types @@ -531,9 +530,10 @@ def prepare_message(email, recipient, recipients_list): fid = attachment.get("fid") if fid: - fname, fcontent = get_file(fid) + _file = frappe.get_doc("File", {"file_name": fid}) + fcontent = _file.get_content() attachment.update({ - 'fname': fname, + 'fname': _file.file_name, 'fcontent': fcontent, 'parent': msg_obj }) diff --git a/frappe/patches/v4_0/file_manager_hooks.py b/frappe/patches/v4_0/file_manager_hooks.py index 018e52ca75..6be3b25124 100644 --- a/frappe/patches/v4_0/file_manager_hooks.py +++ b/frappe/patches/v4_0/file_manager_hooks.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals, print_function import frappe import os from frappe.utils import get_files_path -from frappe.core.doctype.file.file import get_content_hash, get_file +from frappe.core.doctype.file.file import get_content_hash def execute(): @@ -22,7 +22,8 @@ def execute(): else: b.file_url = os.path.normpath('/files/' + old_file_name) try: - _file_name, content = get_file(name) + _file = frappe.get_doc("File", {"file_name": name}) + content = _file.get_content() b.content_hash = get_content_hash(content) except IOError: print('Warning: Error processing ', name) diff --git a/frappe/patches/v4_1/file_manager_fix.py b/frappe/patches/v4_1/file_manager_fix.py index edb450d21a..cd30c94177 100644 --- a/frappe/patches/v4_1/file_manager_fix.py +++ b/frappe/patches/v4_1/file_manager_fix.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals, print_function import frappe import os -from frappe.core.doctype.file.file import get_content_hash, get_file, get_file_name +from frappe.core.doctype.file.file import get_content_hash, get_file_name from frappe.utils import get_files_path, get_site_path # The files missed by the previous patch might have been replaced with new files @@ -36,7 +36,8 @@ def execute(): else: b.file_url = os.path.normpath('/files/' + old_file_name) try: - _file_name, content = get_file(name) + _file = frappe.get_doc("File", {"file_name": name}) + content = _file.get_content() b.content_hash = get_content_hash(content) except IOError: print('Warning: Error processing ', name) diff --git a/frappe/utils/csvutils.py b/frappe/utils/csvutils.py index fcbcf2a2f8..72b03ae9d0 100644 --- a/frappe/utils/csvutils.py +++ b/frappe/utils/csvutils.py @@ -16,7 +16,7 @@ def read_csv_content_from_uploaded_file(ignore_encoding=False): fcontent = upfile.read() else: _file = frappe.new_doc("File") - _, fcontent = _file.get_uploaded_content() + fcontent = _file.get_uploaded_content() return read_csv_content(fcontent, ignore_encoding) def read_csv_content_from_attached_file(doc): @@ -30,8 +30,8 @@ def read_csv_content_from_attached_file(doc): raise Exception try: - from frappe.core.doctype.file.file import get_file - fname, fcontent = get_file(fileid) + _file = frappe.get_doc("File", {"file_name": fileid}) + fcontent = _file.get_content() return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors')) except Exception: frappe.throw(_("Unable to open attached file. Did you export it as CSV?"), title=_('Invalid CSV Format')) diff --git a/frappe/utils/response.py b/frappe/utils/response.py index 78eb88cee1..284d1fbc25 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -17,7 +17,6 @@ from werkzeug.local import LocalProxy from werkzeug.wsgi import wrap_file from werkzeug.wrappers import Response from werkzeug.exceptions import NotFound, Forbidden -from frappe.core.doctype.file.file import check_file_permission from frappe.website.render import render from frappe.utils import cint from six import text_type @@ -147,7 +146,8 @@ def download_backup(path): def download_private_file(path): """Checks permissions and sends back private file""" try: - check_file_permission(path) + _file = frappe.get_doc("File", {"file_url": path}) + _file.check_file_permission() except frappe.PermissionError: raise Forbidden(_("You don't have permission to access this file")) diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index b7c26ba5c3..4354f56d21 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -70,8 +70,8 @@ def handle_html(data): def read_xlsx_file_from_attached_file(file_id=None, fcontent=None, filepath=None): if file_id: - from frappe.core.doctype.file.file import get_file_path - filename = get_file_path(file_id) + _file = frappe.get_doc("File", {"file_name": file_id}) + filename = _file.get_full_path() elif fcontent: from io import BytesIO filename = BytesIO(fcontent)