From b56e6fe4db38e0f1bb445f38d2b807f7a351b3c6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 21 Dec 2021 14:57:09 +0530 Subject: [PATCH] fix: Safe decode base64 data in case of incorrect padding This could possibly happen in case of corrupted/partial files. But for the most part, if there's a missing trailing = or three at the end too. Traceback: Traceback (most recent call last): File "/home/frappe/frappe-bench/apps/frappe/frappe/app.py", line 68, in application response = frappe.api.handle() File "/home/frappe/frappe-bench/apps/frappe/frappe/api.py", line 55, in handle return frappe.handler.handle() File "/home/frappe/frappe-bench/apps/frappe/frappe/handler.py", line 31, in handle data = execute_cmd(cmd) File "/home/frappe/frappe-bench/apps/frappe/frappe/handler.py", line 67, in execute_cmd return frappe.call(method, **frappe.form_dict) File "/home/frappe/frappe-bench/apps/frappe/frappe/__init__.py", line 1208, in call return fn(*args, **newargs) File "/home/frappe/frappe-bench/apps/frappe/frappe/desk/form/save.py", line 21, in savedocs doc.save() File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 285, in save return self._save(*args, **kwargs) File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 307, in _save self.insert() File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 239, in insert self._validate() File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 499, in _validate self._extract_images_from_text_editor() File "/home/frappe/frappe-bench/apps/frappe/frappe/model/base_document.py", line 983, in _extract_images_from_text_editor extract_images_from_doc(self, df.fieldname) File "/home/frappe/frappe-bench/apps/frappe/frappe/core/doctype/file/file.py", line 779, in extract_images_from_doc content = extract_images_from_html(doc, content) File "/home/frappe/frappe-bench/apps/frappe/frappe/core/doctype/file/file.py", line 822, in extract_images_from_html content = re.sub(r']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) File "/home/frappe/frappe-bench/env/lib/python3.6/re.py", line 191, in sub return _compile(pattern, flags).sub(repl, string, count) File "/home/frappe/frappe-bench/apps/frappe/frappe/core/doctype/file/file.py", line 814, in _save_file _file.save(ignore_permissions=True) File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 285, in save return self._save(*args, **kwargs) File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 307, in _save self.insert() File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 231, in insert self.run_method("before_insert") File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 860, in run_method out = Document.hook(fn)(self, *args, **kwargs) File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 1158, in composer return composed(self, method, *args, **kwargs) File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 1141, in runner add_to_return_value(self, fn(self, *args, **kwargs)) File "/home/frappe/frappe-bench/apps/frappe/frappe/model/document.py", line 854, in fn = lambda self, *args, **kwargs: getattr(self, method)(*args, **kwargs) File "/home/frappe/frappe-bench/apps/frappe/frappe/core/doctype/file/file.py", line 56, in before_insert self.save_file(content=self.content, decode=self.decode) File "/home/frappe/frappe-bench/apps/frappe/frappe/core/doctype/file/file.py", line 436, in save_file self.content = base64.b64decode(self.content) File "/home/frappe/frappe-bench/env/lib/python3.6/base64.py", line 87, in b64decode return binascii.a2b_base64(s) binascii.Error: Incorrect padding --- frappe/core/doctype/file/file.py | 5 +++-- frappe/utils/file_manager.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 91090bdd77..adf10b9a03 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -29,6 +29,7 @@ from frappe import _, conf, safe_decode from frappe.model.document import Document from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, get_hook_method, random_string, strip from frappe.utils.image import strip_exif_data, optimize_image +from frappe.utils.file_manager import safe_b64decode class MaxFileSizeReachedError(frappe.ValidationError): pass @@ -436,7 +437,7 @@ class File(Document): if b"," in self.content: self.content = self.content.split(b",")[1] - self.content = base64.b64decode(self.content) + self.content = safe_b64decode(self.content) if not self.is_private: self.is_private = 0 @@ -852,7 +853,7 @@ def extract_images_from_html(doc, content, is_private=False): content = content.encode("utf-8") if b"," in content: content = content.split(b",")[1] - content = base64.b64decode(content) + content = safe_b64decode(content) content = optimize_image(content, mtype) diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index 56394442f3..1e654d7881 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import frappe @@ -17,6 +17,20 @@ class MaxFileSizeReachedError(frappe.ValidationError): pass +def safe_b64decode(binary: bytes) -> bytes: + """Adds padding if doesn't already exist before decoding. + + This attempts to avoid the `binascii.Error: Incorrect padding` error raised + when the number of trailing = is simply not enough :crie:. Although, it may + be an indication of corrupted data. + + Refs: + * https://en.wikipedia.org/wiki/Base64 + * https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding + """ + return base64.b64decode(binary + b"===") + + def get_file_url(file_data_name): data = frappe.db.get_value("File", file_data_name, ["file_name", "file_url"], as_dict=True) return data.file_url or data.file_name @@ -112,7 +126,7 @@ def get_uploaded_content(): if 'filedata' in frappe.form_dict: if "," in frappe.form_dict.filedata: frappe.form_dict.filedata = frappe.form_dict.filedata.rsplit(",", 1)[1] - frappe.uploaded_content = base64.b64decode(frappe.form_dict.filedata) + frappe.uploaded_content = safe_b64decode(frappe.form_dict.filedata) frappe.uploaded_filename = frappe.form_dict.filename return frappe.uploaded_filename, frappe.uploaded_content else: @@ -126,7 +140,7 @@ def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, d if b"," in content: content = content.split(b",")[1] - content = base64.b64decode(content) + content = safe_b64decode(content) file_size = check_max_file_size(content) content_hash = get_content_hash(content)