From 05901488d56d67e3ab9d5543710b97e4b4be7b1b Mon Sep 17 00:00:00 2001 From: Safwan Samsudeen Date: Tue, 10 Mar 2026 12:09:34 +0530 Subject: [PATCH 1/5] feat: add chunking to files --- frappe/handler.py | 23 +- .../js/frappe/file_uploader/FileUploader.vue | 205 +++++++++--------- 2 files changed, 126 insertions(+), 102 deletions(-) diff --git a/frappe/handler.py b/frappe/handler.py index 4f647e0b88..ecdb45ab50 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -3,6 +3,7 @@ import os from mimetypes import guess_type +from pathlib import Path from typing import TYPE_CHECKING from werkzeug.wrappers import Response @@ -15,7 +16,7 @@ from frappe.core.doctype.file.utils import find_file_by_url from frappe.core.doctype.server_script.server_script_utils import get_server_script_map from frappe.monitor import add_data_to_monitor from frappe.permissions import check_doctype_permission -from frappe.utils import cint +from frappe.utils import cint, get_files_path from frappe.utils.csvutils import build_csv_response from frappe.utils.deprecations import deprecated from frappe.utils.image import optimize_image @@ -162,9 +163,27 @@ def upload_file(): if "file" in files: file = files["file"] - content = file.stream.read() filename = file.filename + total_file_size = frappe.form_dict.total_file_size + if frappe.form_dict.chunk_index: + current_chunk = int(frappe.form_dict.chunk_index) + total_chunks = int(frappe.form_dict.total_chunk_count) + offset = int(frappe.form_dict.chunk_byte_offset) + else: + offset = 0 + current_chunk = 0 + total_chunks = 1 + + temp_path = Path(get_files_path("uploads", filename, is_private=is_private)) + with temp_path.open("ab") as f: + f.seek(offset) + f.write(file.stream.read()) + if not f.tell() >= int(total_file_size) or current_chunk != total_chunks - 1: + return + + content = temp_path.read_bytes() + temp_path.unlink() content_type = guess_type(filename)[0] if optimize and content_type and content_type.startswith("image/"): args = {"content": content, "content_type": content_type} diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index f21d925efe..7decdb0019 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -568,79 +568,86 @@ function return_as_dataurl() { close_dialog.value = true; return Promise.all(promises); } -function upload_file(file, i) { + +async function upload_file(file, i) { currently_uploading.value = i; - return new Promise((resolve, reject) => { - let xhr = new XMLHttpRequest(); - xhr.upload.addEventListener("loadstart", (e) => { - file.uploading = true; - }); - xhr.upload.addEventListener("progress", (e) => { - if (e.lengthComputable) { - file.progress = e.loaded; - file.total = e.total; - } - }); - xhr.upload.addEventListener("load", (e) => { - file.uploading = false; - }); - xhr.addEventListener("error", (e) => { - file.failed = true; - reject(); - }); - xhr.onreadystatechange = () => { - if (xhr.readyState == XMLHttpRequest.DONE) { + const CHUNK_SIZE = 500 * 1024; // 500KB chunks + + const use_chunks = file.file_obj && file.file_obj.size > CHUNK_SIZE; + const total_chunks = use_chunks ? Math.ceil(file.file_obj.size / CHUNK_SIZE) : 1; + + const send_chunk = (chunk_blob, chunk_index, chunk_byte_offset) => { + return new Promise((resolve, reject) => { + let xhr = new XMLHttpRequest(); + + xhr.upload.addEventListener("loadstart", () => { + file.uploading = true; + }); + xhr.upload.addEventListener("progress", (e) => { + if (e.lengthComputable) { + // Accumulate progress across chunks + file.progress = chunk_byte_offset + e.loaded; + file.total = file.file_obj.size; + } + }); + xhr.upload.addEventListener("load", () => { + if (chunk_index === total_chunks - 1) { + file.uploading = false; + } + }); + xhr.addEventListener("error", () => { + file.failed = true; + reject(); + }); + xhr.onreadystatechange = () => { + if (xhr.readyState !== XMLHttpRequest.DONE) return; + if (xhr.status === 200) { - resolve(); - file.request_succeeded = true; - let r = null; - let file_doc = null; - try { - r = JSON.parse(xhr.responseText); - if (r.message.doctype === "File") { - file_doc = r.message; + // Only the last chunk returns a meaningful response + if (chunk_index === total_chunks - 1) { + file.request_succeeded = true; + let r = null; + let file_doc = null; + try { + r = JSON.parse(xhr.responseText); + if (r.message?.doctype === "File") { + file_doc = r.message; + } + } catch (e) { + r = xhr.responseText; } - } catch (e) { - r = xhr.responseText; - } - file.doc = file_doc; - - if (props.on_success) { - props.on_success(file_doc, r); - } - - if ( - i == files.value.length - 1 && - files.value.every((file) => file.request_succeeded) - ) { - close_dialog.value = true; - } - if (show_web_link.value && file.file_url) { - close_dialog.value = true; + file.doc = file_doc; + if (props.on_success) { + props.on_success(file_doc, r); + } + if ( + i == files.value.length - 1 && + files.value.every((f) => f.request_succeeded) + ) { + close_dialog.value = true; + } } + resolve(); } else if (xhr.status === 403) { - reject(); file.failed = true; let response = parse_error_response(xhr.responseText); file.error_message = __("Not permitted. {0}.", [response.error_message || ""]); if (response.server_messages.length) { file.error_message += `\n${response.server_messages.join("\n")}`; } - } else if (xhr.status === 413) { reject(); + } else if (xhr.status === 413) { file.failed = true; file.error_message = __("Size exceeds the maximum allowed file size."); - } else if (xhr.status === 417) { reject(); - // regular frappe.throw() in backend + } else if (xhr.status === 417) { file.failed = true; let response = parse_error_response(xhr.responseText); file.error_message = response.server_messages.length ? response.server_messages.join("\n") : __("File upload failed."); - if (show_web_link.value && web_link.value && file.file_url) { web_link.value.invalid_input(__(file.error_message)); } else if (!files.value.includes(file)) { @@ -650,8 +657,8 @@ function upload_file(file, i) { indicator: "red", }); } - } else { reject(); + } else { file.failed = true; let detail = xhr.statusText || @@ -660,7 +667,6 @@ function upload_file(file, i) { xhr.status === 0 ? __("XMLHttpRequest Error") : `${xhr.status} : ${detail}`; - let error = null; try { error = JSON.parse(xhr.responseText); @@ -668,61 +674,60 @@ function upload_file(file, i) { // pass } frappe.request.cleanup({}, error); + reject(); } + }; + + xhr.open("POST", "/api/method/upload_file", true); + xhr.setRequestHeader("Accept", "application/json"); + xhr.setRequestHeader("X-Frappe-CSRF-Token", frappe.csrf_token); + + let form_data = new FormData(); + + if (chunk_blob) { + form_data.append("file", chunk_blob, file.name); } - }; - xhr.open("POST", "/api/method/upload_file", true); - xhr.setRequestHeader("Accept", "application/json"); - xhr.setRequestHeader("X-Frappe-CSRF-Token", frappe.csrf_token); - let form_data = new FormData(); - if (file.file_obj) { - form_data.append("file", file.file_obj, file.name); - } - form_data.append("is_private", +file.private); - form_data.append("folder", props.folder); + form_data.append("is_private", +file.private); + form_data.append("folder", props.folder); + form_data.append("total_file_size", file.file_obj?.size ?? 0); - if (file.file_url) { - form_data.append("file_url", file.file_url); - } - if (file.file_size) { - form_data.append("file_size", file.file_size); - } - if (file.file_name) { - form_data.append("file_name", file.file_name); - } - if (file.library_file_name) { - form_data.append("library_file_name", file.library_file_name); - } + if (use_chunks) { + form_data.append("chunk_index", chunk_index); + form_data.append("total_chunk_count", total_chunks); + form_data.append("chunk_byte_offset", chunk_byte_offset); + } - if (props.doctype) { - form_data.append("doctype", props.doctype); - } + if (file.file_url) form_data.append("file_url", file.file_url); + if (file.file_size) form_data.append("file_size", file.file_size); + if (file.file_name) form_data.append("file_name", file.file_name); + if (file.library_file_name) + form_data.append("library_file_name", file.library_file_name); + if (props.doctype) form_data.append("doctype", props.doctype); + if (props.docname) form_data.append("docname", props.docname); + if (props.fieldname) form_data.append("fieldname", props.fieldname); + if (props.method) form_data.append("method", props.method); + if (file.optimize) form_data.append("optimize", true); + if (props.attach_doc_image) { + form_data.append("max_width", 200); + form_data.append("max_height", 200); + } - if (props.docname) { - form_data.append("docname", props.docname); - } + xhr.send(form_data); + }); + }; - if (props.fieldname) { - form_data.append("fieldname", props.fieldname); - } - - if (props.method) { - form_data.append("method", props.method); - } - - if (file.optimize) { - form_data.append("optimize", true); - } - - if (props.attach_doc_image) { - form_data.append("max_width", 200); - form_data.append("max_height", 200); - } - - xhr.send(form_data); - }); + // Slice and send chunks sequentially + let chunk_byte_offset = 0; + for (let chunk_index = 0; chunk_index < total_chunks; chunk_index++) { + const chunk_blob = file.file_obj + ? file.file_obj.slice(chunk_byte_offset, chunk_byte_offset + CHUNK_SIZE) + : null; + await send_chunk(chunk_blob, chunk_index, chunk_byte_offset); + chunk_byte_offset += CHUNK_SIZE; + } } + function parse_error_response(response_text) { let error_message = ""; let server_messages = []; From 8fe9996e30fbea8c1f16ee8a821974596c56475d Mon Sep 17 00:00:00 2001 From: Safwan Samsudeen Date: Tue, 10 Mar 2026 12:48:26 +0530 Subject: [PATCH 2/5] fix: allow setting chunk size from site settings --- frappe/boot.py | 3 ++- frappe/core/api/file.py | 4 ++++ frappe/public/js/frappe/file_uploader/FileUploader.vue | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/boot.py b/frappe/boot.py index 5042693b77..43f9737929 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -151,9 +151,10 @@ def get_letter_heads(): def load_conf_settings(bootinfo): - from frappe.core.api.file import get_max_file_size + from frappe.core.api.file import get_file_chunk_size, get_max_file_size bootinfo.max_file_size = get_max_file_size() + bootinfo.file_chunk_size = get_file_chunk_size() for key in ("developer_mode", "socketio_port", "file_watcher_port"): if key in frappe.conf: bootinfo[key] = frappe.conf.get(key) diff --git a/frappe/core/api/file.py b/frappe/core/api/file.py index 5e29ba9f99..633a00a8f4 100644 --- a/frappe/core/api/file.py +++ b/frappe/core/api/file.py @@ -90,6 +90,10 @@ def get_max_file_size() -> int: ) +def get_file_chunk_size() -> int: + return cint(frappe.conf.get("file_chunk_size")) or 25 * 1024 * 1024 + + @frappe.whitelist() def create_new_folder(file_name: str, folder: str) -> File: """create new folder under current parent folder""" diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index 7decdb0019..9edfa656c1 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -572,7 +572,7 @@ function return_as_dataurl() { async function upload_file(file, i) { currently_uploading.value = i; - const CHUNK_SIZE = 500 * 1024; // 500KB chunks + const CHUNK_SIZE = frappe.boot.file_chunk_size; const use_chunks = file.file_obj && file.file_obj.size > CHUNK_SIZE; const total_chunks = use_chunks ? Math.ceil(file.file_obj.size / CHUNK_SIZE) : 1; From 1e6855d592668f9c10f31b66dc6ecaf77d469ff6 Mon Sep 17 00:00:00 2001 From: Safwan Samsudeen Date: Tue, 10 Mar 2026 13:10:05 +0530 Subject: [PATCH 3/5] chore: remove unnecessary diff --- .../js/frappe/file_uploader/FileUploader.vue | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index 9edfa656c1..f2bcda100e 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -568,7 +568,6 @@ function return_as_dataurl() { close_dialog.value = true; return Promise.all(promises); } - async function upload_file(file, i) { currently_uploading.value = i; @@ -586,7 +585,6 @@ async function upload_file(file, i) { }); xhr.upload.addEventListener("progress", (e) => { if (e.lengthComputable) { - // Accumulate progress across chunks file.progress = chunk_byte_offset + e.loaded; file.total = file.file_obj.size; } @@ -604,6 +602,7 @@ async function upload_file(file, i) { if (xhr.readyState !== XMLHttpRequest.DONE) return; if (xhr.status === 200) { + resolve(); // Only the last chunk returns a meaningful response if (chunk_index === total_chunks - 1) { file.request_succeeded = true; @@ -629,25 +628,27 @@ async function upload_file(file, i) { close_dialog.value = true; } } - resolve(); } else if (xhr.status === 403) { + reject(); file.failed = true; let response = parse_error_response(xhr.responseText); file.error_message = __("Not permitted. {0}.", [response.error_message || ""]); if (response.server_messages.length) { file.error_message += `\n${response.server_messages.join("\n")}`; } - reject(); } else if (xhr.status === 413) { + reject(); file.failed = true; file.error_message = __("Size exceeds the maximum allowed file size."); - reject(); } else if (xhr.status === 417) { + reject(); + // regular frappe.throw() in backend file.failed = true; let response = parse_error_response(xhr.responseText); file.error_message = response.server_messages.length ? response.server_messages.join("\n") : __("File upload failed."); + if (show_web_link.value && web_link.value && file.file_url) { web_link.value.invalid_input(__(file.error_message)); } else if (!files.value.includes(file)) { @@ -657,8 +658,8 @@ async function upload_file(file, i) { indicator: "red", }); } - reject(); } else { + reject(); file.failed = true; let detail = xhr.statusText || @@ -667,6 +668,7 @@ async function upload_file(file, i) { xhr.status === 0 ? __("XMLHttpRequest Error") : `${xhr.status} : ${detail}`; + let error = null; try { error = JSON.parse(xhr.responseText); @@ -674,7 +676,6 @@ async function upload_file(file, i) { // pass } frappe.request.cleanup({}, error); - reject(); } }; From a1895b05079b91b3275e55685b5e6681fa46a5a5 Mon Sep 17 00:00:00 2001 From: Safwan Samsudeen Date: Tue, 10 Mar 2026 13:20:40 +0530 Subject: [PATCH 4/5] fix: remove uploads dir usage --- frappe/core/doctype/file/file.py | 2 +- frappe/core/doctype/file/utils.py | 4 ++++ frappe/handler.py | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 4e9e06a808..996180c768 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -751,7 +751,7 @@ class File(Document): return self.save_file_on_filesystem() def save_file_on_filesystem(self): - safe_file_name = re.sub(r"[/\\%?#]", "_", self.file_name) + safe_file_name = get_safe_file_name(self.file_name) if self.is_private: self.file_url = f"/private/files/{safe_file_name}" else: diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index 9d2a797ae8..20ad5d7253 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -476,3 +476,7 @@ def find_file_by_url(path: str, name: str | None = None) -> "File" | None: file: File = frappe.get_doc(doctype="File", **file_data) if file.is_downloadable(): return file + + +def get_safe_file_name(file_name: str) -> str: + return re.sub(r"[/\\%?#]", "_", file_name) diff --git a/frappe/handler.py b/frappe/handler.py index ecdb45ab50..86f04f48b9 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -12,7 +12,7 @@ import frappe import frappe.sessions import frappe.utils from frappe import _, is_whitelisted, ping -from frappe.core.doctype.file.utils import find_file_by_url +from frappe.core.doctype.file.utils import find_file_by_url, get_safe_file_name from frappe.core.doctype.server_script.server_script_utils import get_server_script_map from frappe.monitor import add_data_to_monitor from frappe.permissions import check_doctype_permission @@ -175,7 +175,7 @@ def upload_file(): current_chunk = 0 total_chunks = 1 - temp_path = Path(get_files_path("uploads", filename, is_private=is_private)) + temp_path = Path(get_files_path(".temp-" + get_safe_file_name(filename), is_private=is_private)) with temp_path.open("ab") as f: f.seek(offset) f.write(file.stream.read()) From 5a736d1c84ab26f9a864995d94695faa78f69d02 Mon Sep 17 00:00:00 2001 From: Safwan Samsudeen Date: Tue, 10 Mar 2026 16:33:56 +0530 Subject: [PATCH 5/5] fix: dialog not closing for link uploads --- frappe/public/js/frappe/file_uploader/FileUploader.vue | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index f2bcda100e..048f359114 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -586,7 +586,7 @@ async function upload_file(file, i) { xhr.upload.addEventListener("progress", (e) => { if (e.lengthComputable) { file.progress = chunk_byte_offset + e.loaded; - file.total = file.file_obj.size; + file.total = file.file_obj?.size || e.total; } }); xhr.upload.addEventListener("load", () => { @@ -622,8 +622,9 @@ async function upload_file(file, i) { props.on_success(file_doc, r); } if ( - i == files.value.length - 1 && - files.value.every((f) => f.request_succeeded) + (i == files.value.length - 1 && + files.value.every((f) => f.request_succeeded)) || + (show_web_link.value && file.file_url) ) { close_dialog.value = true; }