From 05901488d56d67e3ab9d5543710b97e4b4be7b1b Mon Sep 17 00:00:00 2001 From: Safwan Samsudeen Date: Tue, 10 Mar 2026 12:09:34 +0530 Subject: [PATCH] feat: add chunking to files --- frappe/handler.py | 23 +- .../js/frappe/file_uploader/FileUploader.vue | 205 +++++++++--------- 2 files changed, 126 insertions(+), 102 deletions(-) diff --git a/frappe/handler.py b/frappe/handler.py index 4f647e0b88..ecdb45ab50 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -3,6 +3,7 @@ import os from mimetypes import guess_type +from pathlib import Path from typing import TYPE_CHECKING from werkzeug.wrappers import Response @@ -15,7 +16,7 @@ from frappe.core.doctype.file.utils import find_file_by_url from frappe.core.doctype.server_script.server_script_utils import get_server_script_map from frappe.monitor import add_data_to_monitor from frappe.permissions import check_doctype_permission -from frappe.utils import cint +from frappe.utils import cint, get_files_path from frappe.utils.csvutils import build_csv_response from frappe.utils.deprecations import deprecated from frappe.utils.image import optimize_image @@ -162,9 +163,27 @@ def upload_file(): if "file" in files: file = files["file"] - content = file.stream.read() filename = file.filename + total_file_size = frappe.form_dict.total_file_size + if frappe.form_dict.chunk_index: + current_chunk = int(frappe.form_dict.chunk_index) + total_chunks = int(frappe.form_dict.total_chunk_count) + offset = int(frappe.form_dict.chunk_byte_offset) + else: + offset = 0 + current_chunk = 0 + total_chunks = 1 + + temp_path = Path(get_files_path("uploads", filename, is_private=is_private)) + with temp_path.open("ab") as f: + f.seek(offset) + f.write(file.stream.read()) + if not f.tell() >= int(total_file_size) or current_chunk != total_chunks - 1: + return + + content = temp_path.read_bytes() + temp_path.unlink() content_type = guess_type(filename)[0] if optimize and content_type and content_type.startswith("image/"): args = {"content": content, "content_type": content_type} diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index f21d925efe..7decdb0019 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -568,79 +568,86 @@ function return_as_dataurl() { close_dialog.value = true; return Promise.all(promises); } -function upload_file(file, i) { + +async function upload_file(file, i) { currently_uploading.value = i; - return new Promise((resolve, reject) => { - let xhr = new XMLHttpRequest(); - xhr.upload.addEventListener("loadstart", (e) => { - file.uploading = true; - }); - xhr.upload.addEventListener("progress", (e) => { - if (e.lengthComputable) { - file.progress = e.loaded; - file.total = e.total; - } - }); - xhr.upload.addEventListener("load", (e) => { - file.uploading = false; - }); - xhr.addEventListener("error", (e) => { - file.failed = true; - reject(); - }); - xhr.onreadystatechange = () => { - if (xhr.readyState == XMLHttpRequest.DONE) { + const CHUNK_SIZE = 500 * 1024; // 500KB chunks + + const use_chunks = file.file_obj && file.file_obj.size > CHUNK_SIZE; + const total_chunks = use_chunks ? Math.ceil(file.file_obj.size / CHUNK_SIZE) : 1; + + const send_chunk = (chunk_blob, chunk_index, chunk_byte_offset) => { + return new Promise((resolve, reject) => { + let xhr = new XMLHttpRequest(); + + xhr.upload.addEventListener("loadstart", () => { + file.uploading = true; + }); + xhr.upload.addEventListener("progress", (e) => { + if (e.lengthComputable) { + // Accumulate progress across chunks + file.progress = chunk_byte_offset + e.loaded; + file.total = file.file_obj.size; + } + }); + xhr.upload.addEventListener("load", () => { + if (chunk_index === total_chunks - 1) { + file.uploading = false; + } + }); + xhr.addEventListener("error", () => { + file.failed = true; + reject(); + }); + xhr.onreadystatechange = () => { + if (xhr.readyState !== XMLHttpRequest.DONE) return; + if (xhr.status === 200) { - resolve(); - file.request_succeeded = true; - let r = null; - let file_doc = null; - try { - r = JSON.parse(xhr.responseText); - if (r.message.doctype === "File") { - file_doc = r.message; + // Only the last chunk returns a meaningful response + if (chunk_index === total_chunks - 1) { + file.request_succeeded = true; + let r = null; + let file_doc = null; + try { + r = JSON.parse(xhr.responseText); + if (r.message?.doctype === "File") { + file_doc = r.message; + } + } catch (e) { + r = xhr.responseText; } - } catch (e) { - r = xhr.responseText; - } - file.doc = file_doc; - - if (props.on_success) { - props.on_success(file_doc, r); - } - - if ( - i == files.value.length - 1 && - files.value.every((file) => file.request_succeeded) - ) { - close_dialog.value = true; - } - if (show_web_link.value && file.file_url) { - close_dialog.value = true; + file.doc = file_doc; + if (props.on_success) { + props.on_success(file_doc, r); + } + if ( + i == files.value.length - 1 && + files.value.every((f) => f.request_succeeded) + ) { + close_dialog.value = true; + } } + resolve(); } else if (xhr.status === 403) { - reject(); file.failed = true; let response = parse_error_response(xhr.responseText); file.error_message = __("Not permitted. {0}.", [response.error_message || ""]); if (response.server_messages.length) { file.error_message += `\n${response.server_messages.join("\n")}`; } - } else if (xhr.status === 413) { reject(); + } else if (xhr.status === 413) { file.failed = true; file.error_message = __("Size exceeds the maximum allowed file size."); - } else if (xhr.status === 417) { reject(); - // regular frappe.throw() in backend + } else if (xhr.status === 417) { file.failed = true; let response = parse_error_response(xhr.responseText); file.error_message = response.server_messages.length ? response.server_messages.join("\n") : __("File upload failed."); - if (show_web_link.value && web_link.value && file.file_url) { web_link.value.invalid_input(__(file.error_message)); } else if (!files.value.includes(file)) { @@ -650,8 +657,8 @@ function upload_file(file, i) { indicator: "red", }); } - } else { reject(); + } else { file.failed = true; let detail = xhr.statusText || @@ -660,7 +667,6 @@ function upload_file(file, i) { xhr.status === 0 ? __("XMLHttpRequest Error") : `${xhr.status} : ${detail}`; - let error = null; try { error = JSON.parse(xhr.responseText); @@ -668,61 +674,60 @@ function upload_file(file, i) { // pass } frappe.request.cleanup({}, error); + reject(); } + }; + + xhr.open("POST", "/api/method/upload_file", true); + xhr.setRequestHeader("Accept", "application/json"); + xhr.setRequestHeader("X-Frappe-CSRF-Token", frappe.csrf_token); + + let form_data = new FormData(); + + if (chunk_blob) { + form_data.append("file", chunk_blob, file.name); } - }; - xhr.open("POST", "/api/method/upload_file", true); - xhr.setRequestHeader("Accept", "application/json"); - xhr.setRequestHeader("X-Frappe-CSRF-Token", frappe.csrf_token); - let form_data = new FormData(); - if (file.file_obj) { - form_data.append("file", file.file_obj, file.name); - } - form_data.append("is_private", +file.private); - form_data.append("folder", props.folder); + form_data.append("is_private", +file.private); + form_data.append("folder", props.folder); + form_data.append("total_file_size", file.file_obj?.size ?? 0); - if (file.file_url) { - form_data.append("file_url", file.file_url); - } - if (file.file_size) { - form_data.append("file_size", file.file_size); - } - if (file.file_name) { - form_data.append("file_name", file.file_name); - } - if (file.library_file_name) { - form_data.append("library_file_name", file.library_file_name); - } + if (use_chunks) { + form_data.append("chunk_index", chunk_index); + form_data.append("total_chunk_count", total_chunks); + form_data.append("chunk_byte_offset", chunk_byte_offset); + } - if (props.doctype) { - form_data.append("doctype", props.doctype); - } + if (file.file_url) form_data.append("file_url", file.file_url); + if (file.file_size) form_data.append("file_size", file.file_size); + if (file.file_name) form_data.append("file_name", file.file_name); + if (file.library_file_name) + form_data.append("library_file_name", file.library_file_name); + if (props.doctype) form_data.append("doctype", props.doctype); + if (props.docname) form_data.append("docname", props.docname); + if (props.fieldname) form_data.append("fieldname", props.fieldname); + if (props.method) form_data.append("method", props.method); + if (file.optimize) form_data.append("optimize", true); + if (props.attach_doc_image) { + form_data.append("max_width", 200); + form_data.append("max_height", 200); + } - if (props.docname) { - form_data.append("docname", props.docname); - } + xhr.send(form_data); + }); + }; - if (props.fieldname) { - form_data.append("fieldname", props.fieldname); - } - - if (props.method) { - form_data.append("method", props.method); - } - - if (file.optimize) { - form_data.append("optimize", true); - } - - if (props.attach_doc_image) { - form_data.append("max_width", 200); - form_data.append("max_height", 200); - } - - xhr.send(form_data); - }); + // Slice and send chunks sequentially + let chunk_byte_offset = 0; + for (let chunk_index = 0; chunk_index < total_chunks; chunk_index++) { + const chunk_blob = file.file_obj + ? file.file_obj.slice(chunk_byte_offset, chunk_byte_offset + CHUNK_SIZE) + : null; + await send_chunk(chunk_blob, chunk_index, chunk_byte_offset); + chunk_byte_offset += CHUNK_SIZE; + } } + function parse_error_response(response_text) { let error_message = ""; let server_messages = [];