file-api: migration improvements and fixes
* migrate more functions to file class * add get_content(), returns file content from file_name * move get_file_path() to get_full_path() to decrease naming ambiguity Signed-off-by: Chinmay Pai <chinmaydpai@gmail.com>
This commit is contained in:
Chinmay Pai
parent
a80c23c9be
commit
8943f6cfd5
12 changed files with 141 additions and 136 deletions
|
|
@ -10,7 +10,6 @@ from email.utils import formataddr
|
|||
from frappe.core.utils import get_parent_doc
|
||||
from frappe.utils import (get_url, get_formatted_email, cint,
|
||||
validate_email_add, split_emails, time_diff_in_seconds, parse_addr, get_datetime)
|
||||
from frappe.core.doctype.file.file import get_file
|
||||
from frappe.email.queue import check_email_limit
|
||||
from frappe.utils.scheduler import log
|
||||
from frappe.email.email_body import get_message_id
|
||||
|
|
@ -285,7 +284,8 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None)
|
|||
# is it a filename?
|
||||
try:
|
||||
# keep this for error handling
|
||||
file = get_file(a)
|
||||
_file = frappe.get_doc("File", {"file_name": a})
|
||||
content = _file.get_content()
|
||||
# these attachments will be attached on-demand
|
||||
# and won't be stored in the message
|
||||
doc.attachments.append({"fid": a})
|
||||
|
|
|
|||
|
|
@ -272,9 +272,9 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
|
|||
# header
|
||||
filename, file_extension = ['','']
|
||||
if not rows:
|
||||
from frappe.core.doctype.file.file import get_file # get_file_doc
|
||||
fname, fcontent = get_file(data_import_doc.import_file)
|
||||
filename, file_extension = os.path.splitext(fname)
|
||||
_file = frappe.get_doc("File", {"file_name": data_import_doc.import_file})
|
||||
fcontent = _file.get_content()
|
||||
filename, file_extension = os.path.splitext(_file.file_name)
|
||||
|
||||
if file_extension == '.xlsx' and from_data_import == 'Yes':
|
||||
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
|
||||
|
|
|
|||
|
|
@ -345,6 +345,61 @@ class File(NestedSet):
|
|||
"comment": comment.as_dict() if comment else {}
|
||||
}
|
||||
|
||||
def get_content(self):
|
||||
"""Returns [`file_name`, `content`] for given file name `fname`"""
|
||||
if self.get('content'):
|
||||
return self.content
|
||||
file_path = self.get_full_path()
|
||||
|
||||
# read the file
|
||||
if PY2:
|
||||
with open(encode(file_path)) as f:
|
||||
content = f.read()
|
||||
else:
|
||||
with io.open(encode(file_path), mode='rb') as f:
|
||||
content = f.read()
|
||||
try:
|
||||
# for plain text files
|
||||
content = content.decode()
|
||||
except UnicodeDecodeError:
|
||||
# for .png, .jpg, etc
|
||||
pass
|
||||
|
||||
return content
|
||||
|
||||
def get_full_path(self):
|
||||
"""Returns file path from given file name"""
|
||||
|
||||
file_path = self.file_url
|
||||
|
||||
if "/" not in file_path:
|
||||
file_path = "/files/" + file_path
|
||||
|
||||
if file_path.startswith("/private/files/"):
|
||||
file_path = get_files_path(*file_path.split("/private/files/", 1)[1].split("/"), is_private=1)
|
||||
|
||||
elif file_path.startswith("/files/"):
|
||||
file_path = get_files_path(*file_path.split("/files/", 1)[1].split("/"))
|
||||
|
||||
else:
|
||||
frappe.throw(_("There is some problem with the file url: {0}").format(file_path))
|
||||
|
||||
return file_path
|
||||
|
||||
def write_file(self):
|
||||
"""write file to disk with a random name (to compare)"""
|
||||
file_path = get_files_path(is_private=self.is_private)
|
||||
|
||||
# create directory (if not exists)
|
||||
frappe.create_folder(file_path)
|
||||
# write the file
|
||||
self.content = self.get_content()
|
||||
if isinstance(self.content, text_type):
|
||||
self.content = self.content.encode()
|
||||
with open(os.path.join(file_path.encode('utf-8'), self.file_name.encode('utf-8')), 'wb+') as f:
|
||||
f.write(self.content)
|
||||
|
||||
return get_files_path(self.file_name, is_private=self.is_private)
|
||||
|
||||
def get_file_doc(self):
|
||||
'''returns File object (Document) from given parameters or form_dict'''
|
||||
|
|
@ -368,7 +423,7 @@ class File(NestedSet):
|
|||
|
||||
|
||||
def save_uploaded(self):
|
||||
self.file_name, self.content = self.get_uploaded_content()
|
||||
self.content = self.get_uploaded_content()
|
||||
if self.content:
|
||||
return self.save_file(content=self.content)
|
||||
else:
|
||||
|
|
@ -408,12 +463,11 @@ class File(NestedSet):
|
|||
if "," in frappe.form_dict.filedata:
|
||||
frappe.form_dict.filedata = frappe.form_dict.filedata.rsplit(",", 1)[1]
|
||||
frappe.uploaded_content = base64.b64decode(frappe.form_dict.filedata)
|
||||
frappe.uploaded_filename = frappe.form_dict.filename
|
||||
return frappe.uploaded_filename, frappe.uploaded_content
|
||||
elif self.content and self.file_name:
|
||||
return self.file_name, self.content
|
||||
return frappe.uploaded_content
|
||||
elif self.content:
|
||||
return self.content
|
||||
frappe.msgprint(_('No file attached'))
|
||||
return None, None
|
||||
return None
|
||||
|
||||
|
||||
def save_file(self, content=None, decode=False):
|
||||
|
|
@ -436,8 +490,11 @@ class File(NestedSet):
|
|||
if not file_data:
|
||||
call_hook_method("before_write_file", file_size=self.file_size)
|
||||
|
||||
write_file_method = get_hook_method('write_file', fallback=save_file_on_filesystem)
|
||||
file_data = write_file_method(self.file_name, self.content, self.content_type, self.is_private)
|
||||
write_file_method = get_hook_method('write_file')
|
||||
if write_file_method:
|
||||
file_data = write_file_method(self)
|
||||
else:
|
||||
file_data = self.save_file_on_filesystem()
|
||||
file_data = copy(file_data)
|
||||
|
||||
file_data.update({
|
||||
|
|
@ -460,6 +517,18 @@ class File(NestedSet):
|
|||
|
||||
return f
|
||||
|
||||
def save_file_on_filesystem(self):
|
||||
fpath = self.write_file()
|
||||
|
||||
if self.is_private:
|
||||
self.file_url = "/private/files/{0}".format(self.file_name)
|
||||
else:
|
||||
self.file_url = "/files/{0}".format(self.file_name)
|
||||
|
||||
return {
|
||||
'file_name': os.path.basename(fpath),
|
||||
'file_url': self.file_url
|
||||
}
|
||||
|
||||
def get_file_data_from_hash(self):
|
||||
for name in frappe.db.sql_list("select name from `tabFile` where content_hash=%s and is_private=%s",
|
||||
|
|
@ -482,8 +551,31 @@ class File(NestedSet):
|
|||
|
||||
|
||||
def delete_file_data_content(self, only_thumbnail=False):
|
||||
method = get_hook_method('delete_file_data_content', fallback=delete_file_from_filesystem)
|
||||
method(self, only_thumbnail=only_thumbnail)
|
||||
method = get_hook_method('delete_file_data_content')
|
||||
if method:
|
||||
method(self, only_thumbnail=only_thumbnail)
|
||||
else:
|
||||
self.delete_file_from_filesystem(only_thumbnail=only_thumbnail)
|
||||
|
||||
|
||||
def delete_file_from_filesystem(self, only_thumbnail=False):
|
||||
"""Delete file, thumbnail from File document"""
|
||||
if only_thumbnail:
|
||||
delete_file(self.thumbnail_url)
|
||||
else:
|
||||
delete_file(self.file_url)
|
||||
delete_file(self.thumbnail_url)
|
||||
|
||||
|
||||
def check_file_permission(self):
|
||||
for file in frappe.get_all("File", filters={"file_url": self.file_url, "is_private": 1},
|
||||
fields=["name", "attached_to_doctype", "attached_to_name"]):
|
||||
|
||||
if (frappe.has_permission("File", ptype="read", doc=file.name)
|
||||
or frappe.has_permission(file.attached_to_doctype, ptype="read", doc=file.attached_to_name)):
|
||||
return True
|
||||
|
||||
raise frappe.PermissionError
|
||||
|
||||
|
||||
def on_doctype_update():
|
||||
|
|
@ -615,44 +707,6 @@ def get_web_image(file_url):
|
|||
return image, filename, extn
|
||||
|
||||
|
||||
def save_file_on_filesystem(file_name, content, content_type, is_private):
|
||||
fpath = write_file(file_name, content, is_private=is_private)
|
||||
|
||||
if is_private:
|
||||
file_url = "/private/files/{0}".format(file_name)
|
||||
else:
|
||||
file_url = "/files/{0}".format(file_name)
|
||||
|
||||
return {
|
||||
'file_name': os.path.basename(fpath),
|
||||
'file_url': file_url
|
||||
}
|
||||
|
||||
|
||||
def write_file(file_name, content, is_private=0):
|
||||
"""write file to disk with a random name (to compare)"""
|
||||
file_path = get_files_path(is_private=is_private)
|
||||
|
||||
# create directory (if not exists)
|
||||
frappe.create_folder(file_path)
|
||||
# write the file
|
||||
if isinstance(content, text_type):
|
||||
content = content.encode()
|
||||
with open(os.path.join(file_path.encode('utf-8'), file_name.encode('utf-8')), 'wb+') as f:
|
||||
f.write(content)
|
||||
|
||||
return get_files_path(file_name, is_private=is_private)
|
||||
|
||||
|
||||
def delete_file_from_filesystem(doc, only_thumbnail=False):
|
||||
"""Delete file, thumbnail from File document"""
|
||||
if only_thumbnail:
|
||||
delete_file(doc.thumbnail_url)
|
||||
else:
|
||||
delete_file(doc.file_url)
|
||||
delete_file(doc.thumbnail_url)
|
||||
|
||||
|
||||
def delete_file(path):
|
||||
"""Delete file from `public folder`"""
|
||||
if path:
|
||||
|
|
@ -719,51 +773,6 @@ def remove_file_by_url(file_url, doctype=None, name=None):
|
|||
return remove_file(fid=fid)
|
||||
|
||||
|
||||
def get_file(fname):
|
||||
"""Returns [`file_name`, `content`] for given file name `fname`"""
|
||||
file_path = get_file_path(fname)
|
||||
|
||||
# read the file
|
||||
if PY2:
|
||||
with open(encode(file_path)) as f:
|
||||
content = f.read()
|
||||
else:
|
||||
with io.open(encode(file_path), mode='rb') as f:
|
||||
content = f.read()
|
||||
try:
|
||||
# for plain text files
|
||||
content = content.decode()
|
||||
except UnicodeDecodeError:
|
||||
# for .png, .jpg, etc
|
||||
pass
|
||||
|
||||
return [file_path.rsplit("/", 1)[-1], content]
|
||||
|
||||
|
||||
def get_file_path(file_name):
|
||||
"""Returns file path from given file name"""
|
||||
f = frappe.db.sql("""select file_url from `tabFile`
|
||||
where name=%s or file_name=%s""", (file_name, file_name))
|
||||
if f:
|
||||
file_name = f[0][0]
|
||||
|
||||
file_path = file_name
|
||||
|
||||
if "/" not in file_path:
|
||||
file_path = "/files/" + file_path
|
||||
|
||||
if file_path.startswith("/private/files/"):
|
||||
file_path = get_files_path(*file_path.split("/private/files/", 1)[1].split("/"), is_private=1)
|
||||
|
||||
elif file_path.startswith("/files/"):
|
||||
file_path = get_files_path(*file_path.split("/files/", 1)[1].split("/"))
|
||||
|
||||
else:
|
||||
frappe.throw(_("There is some problem with the file url: {0}").format(file_path))
|
||||
|
||||
return file_path
|
||||
|
||||
|
||||
def get_content_hash(content):
|
||||
if isinstance(content, text_type):
|
||||
content = content.encode()
|
||||
|
|
@ -795,7 +804,7 @@ def download_file(file_url):
|
|||
Endpoint : frappe.core.doctype.file.file.download_file
|
||||
URL Params : file_name = /path/to/file relative to site path
|
||||
"""
|
||||
file_doc = frappe.get_doc("File", {"file_url":file_url})
|
||||
file_doc = frappe.get_doc("File", {"file_url": file_url})
|
||||
file_doc.check_permission("read")
|
||||
path = os.path.join(get_files_path(), os.path.basename(file_url))
|
||||
|
||||
|
|
@ -859,16 +868,6 @@ def get_random_filename(extn=None, content_type=None):
|
|||
return random_string(7) + (extn or "")
|
||||
|
||||
|
||||
def check_file_permission(file_url):
|
||||
for file in frappe.get_all("File", filters={"file_url": file_url, "is_private": 1}, fields=["name", "attached_to_doctype", "attached_to_name"]):
|
||||
|
||||
if (frappe.has_permission("File", ptype="read", doc=file.name)
|
||||
or frappe.has_permission(file.attached_to_doctype, ptype="read", doc=file.attached_to_name)):
|
||||
return True
|
||||
|
||||
raise frappe.PermissionError
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def unzip_file(name):
|
||||
'''Unzip the given file and make file records for each of the extracted files'''
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import frappe
|
|||
import os
|
||||
import unittest
|
||||
from frappe import _
|
||||
from frappe.core.doctype.file.file import move_file, get_file, get_files_path
|
||||
from frappe.core.doctype.file.file import move_file, get_files_path
|
||||
# test_records = frappe.get_test_records('File')
|
||||
|
||||
test_content1 = 'Hello'
|
||||
|
|
@ -33,7 +33,8 @@ class TestSimpleFile(unittest.TestCase):
|
|||
self.saved_filename = get_files_path(self.saved_file.file_name)
|
||||
|
||||
def test_save(self):
|
||||
_, content = get_file(self.saved_file.name)
|
||||
_file = frappe.get_doc("File", {"file_name": self.saved_file.file_name})
|
||||
content = _file.get_content()
|
||||
self.assertEqual(content, self.test_content)
|
||||
|
||||
def tearDown(self):
|
||||
|
|
@ -61,9 +62,11 @@ class TestSameFileName(unittest.TestCase):
|
|||
self.saved_filename2 = get_files_path(self.saved_file2.file_name)
|
||||
|
||||
def test_saved_content(self):
|
||||
_, content1 = get_file(self.saved_file1.name)
|
||||
_file = frappe.get_doc("File", {"file_name": self.saved_file1.file_name})
|
||||
content1 = _file.get_content()
|
||||
self.assertEqual(content1, self.test_content1)
|
||||
_, content2 = get_file(self.saved_file2.name)
|
||||
_file = frappe.get_doc("File", {"file_name": self.saved_file2.file_name})
|
||||
content2 = _file.get_content()
|
||||
self.assertEqual(content2, self.test_content2)
|
||||
|
||||
def tearDown(self):
|
||||
|
|
@ -94,8 +97,10 @@ class TestSameContent(unittest.TestCase):
|
|||
self.saved_filename2 = get_files_path(self.saved_file2.file_name)
|
||||
|
||||
def test_saved_content(self):
|
||||
filename1, content1 = get_file(self.saved_file1.name)
|
||||
filename2, content2 = get_file(self.saved_file2.name)
|
||||
_file1 = frappe.get_doc("File", {"file_name": self.saved_file1.file_name})
|
||||
filename1 = _file1.file_name
|
||||
_file2 = frappe.get_doc("File", {"file_name": self.saved_file2.file_name})
|
||||
filename2 = _file2.file_name
|
||||
self.assertEqual(filename1, filename2)
|
||||
self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ test_records = frappe.get_test_records('Email Account')
|
|||
|
||||
from frappe.core.doctype.communication.email import make
|
||||
from frappe.desk.form.load import get_attachments
|
||||
from frappe.core.doctype.file.file import delete_file_from_filesystem
|
||||
from frappe.email.doctype.email_account.email_account import notify_unreplied
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
|
@ -55,7 +54,7 @@ class TestEmailAccount(unittest.TestCase):
|
|||
frappe.db.sql("delete from tabCommunication where sender='test_sender@example.com'")
|
||||
existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'})
|
||||
frappe.delete_doc("File", existing_file.name)
|
||||
delete_file_from_filesystem(existing_file)
|
||||
existing_file.delete_file_from_filesystem()
|
||||
|
||||
with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile:
|
||||
test_mails = [testfile.read()]
|
||||
|
|
@ -73,7 +72,7 @@ class TestEmailAccount(unittest.TestCase):
|
|||
# cleanup
|
||||
existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'})
|
||||
frappe.delete_doc("File", existing_file.name)
|
||||
delete_file_from_filesystem(existing_file)
|
||||
existing_file.delete_file_from_filesystem()
|
||||
|
||||
|
||||
def test_incoming_attached_email_from_outlook_plain_text_only(self):
|
||||
|
|
|
|||
|
|
@ -144,12 +144,12 @@ class EMail:
|
|||
|
||||
def attach_file(self, n):
|
||||
"""attach a file from the `FileData` table"""
|
||||
from frappe.core.doctype.file.file import get_file
|
||||
res = get_file(n)
|
||||
if not res:
|
||||
_file = frappe.get_doc("File", {"file_name": n})
|
||||
content = _file.get_content()
|
||||
if not content:
|
||||
return
|
||||
|
||||
self.add_attachment(res[0], res[1])
|
||||
self.add_attachment(_file.file_name, content)
|
||||
|
||||
def add_attachment(self, fname, fcontent, content_type=None,
|
||||
parent=None, content_id=None, inline=False):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from frappe.email.email_body import get_email, get_formatted_html, add_attachmen
|
|||
from frappe.utils.verified_command import get_signed_params, verify_request
|
||||
from html2text import html2text
|
||||
from frappe.utils import get_url, nowdate, encode, now_datetime, add_days, split_emails, cstr, cint
|
||||
from frappe.core.doctype.file.file import get_file
|
||||
from rq.timeouts import JobTimeoutException
|
||||
from frappe.utils.scheduler import log
|
||||
from six import text_type, string_types
|
||||
|
|
@ -531,9 +530,10 @@ def prepare_message(email, recipient, recipients_list):
|
|||
|
||||
fid = attachment.get("fid")
|
||||
if fid:
|
||||
fname, fcontent = get_file(fid)
|
||||
_file = frappe.get_doc("File", {"file_name": fid})
|
||||
fcontent = _file.get_content()
|
||||
attachment.update({
|
||||
'fname': fname,
|
||||
'fname': _file.file_name,
|
||||
'fcontent': fcontent,
|
||||
'parent': msg_obj
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from __future__ import unicode_literals, print_function
|
|||
import frappe
|
||||
import os
|
||||
from frappe.utils import get_files_path
|
||||
from frappe.core.doctype.file.file import get_content_hash, get_file
|
||||
from frappe.core.doctype.file.file import get_content_hash
|
||||
|
||||
|
||||
def execute():
|
||||
|
|
@ -22,7 +22,8 @@ def execute():
|
|||
else:
|
||||
b.file_url = os.path.normpath('/files/' + old_file_name)
|
||||
try:
|
||||
_file_name, content = get_file(name)
|
||||
_file = frappe.get_doc("File", {"file_name": name})
|
||||
content = _file.get_content()
|
||||
b.content_hash = get_content_hash(content)
|
||||
except IOError:
|
||||
print('Warning: Error processing ', name)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from __future__ import unicode_literals, print_function
|
|||
|
||||
import frappe
|
||||
import os
|
||||
from frappe.core.doctype.file.file import get_content_hash, get_file, get_file_name
|
||||
from frappe.core.doctype.file.file import get_content_hash, get_file_name
|
||||
from frappe.utils import get_files_path, get_site_path
|
||||
|
||||
# The files missed by the previous patch might have been replaced with new files
|
||||
|
|
@ -36,7 +36,8 @@ def execute():
|
|||
else:
|
||||
b.file_url = os.path.normpath('/files/' + old_file_name)
|
||||
try:
|
||||
_file_name, content = get_file(name)
|
||||
_file = frappe.get_doc("File", {"file_name": name})
|
||||
content = _file.get_content()
|
||||
b.content_hash = get_content_hash(content)
|
||||
except IOError:
|
||||
print('Warning: Error processing ', name)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def read_csv_content_from_uploaded_file(ignore_encoding=False):
|
|||
fcontent = upfile.read()
|
||||
else:
|
||||
_file = frappe.new_doc("File")
|
||||
_, fcontent = _file.get_uploaded_content()
|
||||
fcontent = _file.get_uploaded_content()
|
||||
return read_csv_content(fcontent, ignore_encoding)
|
||||
|
||||
def read_csv_content_from_attached_file(doc):
|
||||
|
|
@ -30,8 +30,8 @@ def read_csv_content_from_attached_file(doc):
|
|||
raise Exception
|
||||
|
||||
try:
|
||||
from frappe.core.doctype.file.file import get_file
|
||||
fname, fcontent = get_file(fileid)
|
||||
_file = frappe.get_doc("File", {"file_name": fileid})
|
||||
fcontent = _file.get_content()
|
||||
return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors'))
|
||||
except Exception:
|
||||
frappe.throw(_("Unable to open attached file. Did you export it as CSV?"), title=_('Invalid CSV Format'))
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from werkzeug.local import LocalProxy
|
|||
from werkzeug.wsgi import wrap_file
|
||||
from werkzeug.wrappers import Response
|
||||
from werkzeug.exceptions import NotFound, Forbidden
|
||||
from frappe.core.doctype.file.file import check_file_permission
|
||||
from frappe.website.render import render
|
||||
from frappe.utils import cint
|
||||
from six import text_type
|
||||
|
|
@ -147,7 +146,8 @@ def download_backup(path):
|
|||
def download_private_file(path):
|
||||
"""Checks permissions and sends back private file"""
|
||||
try:
|
||||
check_file_permission(path)
|
||||
_file = frappe.get_doc("File", {"file_url": path})
|
||||
_file.check_file_permission()
|
||||
|
||||
except frappe.PermissionError:
|
||||
raise Forbidden(_("You don't have permission to access this file"))
|
||||
|
|
|
|||
|
|
@ -70,8 +70,8 @@ def handle_html(data):
|
|||
|
||||
def read_xlsx_file_from_attached_file(file_id=None, fcontent=None, filepath=None):
|
||||
if file_id:
|
||||
from frappe.core.doctype.file.file import get_file_path
|
||||
filename = get_file_path(file_id)
|
||||
_file = frappe.get_doc("File", {"file_name": file_id})
|
||||
filename = _file.get_full_path()
|
||||
elif fcontent:
|
||||
from io import BytesIO
|
||||
filename = BytesIO(fcontent)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue