vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: add chunking to files

Browse source
This commit is contained in:
Safwan Samsudeen 2026-03-10 12:09:34 +05:30
parent 4de1e402c5
commit 05901488d5
2 changed files with 126 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
frappe/handler.py
View file

@ -3,6 +3,7 @@
import os
from mimetypes import guess_type
from pathlib import Path
from typing import TYPE_CHECKING
from werkzeug.wrappers import Response
@ -15,7 +16,7 @@ from frappe.core.doctype.file.utils import find_file_by_url
from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
from frappe.monitor import add_data_to_monitor
from frappe.permissions import check_doctype_permission
from frappe.utils import cint
from frappe.utils import cint, get_files_path
from frappe.utils.csvutils import build_csv_response
from frappe.utils.deprecations import deprecated
from frappe.utils.image import optimize_image
@ -162,9 +163,27 @@ def upload_file():
if "file" in files:
file = files["file"]
content = file.stream.read()
filename = file.filename
total_file_size = frappe.form_dict.total_file_size
if frappe.form_dict.chunk_index:
current_chunk = int(frappe.form_dict.chunk_index)
total_chunks = int(frappe.form_dict.total_chunk_count)
offset = int(frappe.form_dict.chunk_byte_offset)
else:
offset = 0
current_chunk = 0
total_chunks = 1
temp_path = Path(get_files_path("uploads", filename, is_private=is_private))
with temp_path.open("ab") as f:
f.seek(offset)
f.write(file.stream.read())
if not f.tell() >= int(total_file_size) or current_chunk != total_chunks - 1:
return
content = temp_path.read_bytes()
temp_path.unlink()
content_type = guess_type(filename)[0]
if optimize and content_type and content_type.startswith("image/"):
args = {"content": content, "content_type": content_type}

205
frappe/public/js/frappe/file_uploader/FileUploader.vue
View file

@ -568,79 +568,86 @@ function return_as_dataurl() {
close_dialog.value = true;
return Promise.all(promises);
}
function upload_file(file, i) {
async function upload_file(file, i) {
currently_uploading.value = i;
return new Promise((resolve, reject) => {
let xhr = new XMLHttpRequest();
xhr.upload.addEventListener("loadstart", (e) => {
file.uploading = true;
});
xhr.upload.addEventListener("progress", (e) => {
if (e.lengthComputable) {
file.progress = e.loaded;
file.total = e.total;
}
});
xhr.upload.addEventListener("load", (e) => {
file.uploading = false;
});
xhr.addEventListener("error", (e) => {
file.failed = true;
reject();
});
xhr.onreadystatechange = () => {
if (xhr.readyState == XMLHttpRequest.DONE) {
const CHUNK_SIZE = 500 * 1024; // 500KB chunks
const use_chunks = file.file_obj && file.file_obj.size > CHUNK_SIZE;
const total_chunks = use_chunks ? Math.ceil(file.file_obj.size / CHUNK_SIZE) : 1;
const send_chunk = (chunk_blob, chunk_index, chunk_byte_offset) => {
return new Promise((resolve, reject) => {
let xhr = new XMLHttpRequest();
xhr.upload.addEventListener("loadstart", () => {
file.uploading = true;
});
xhr.upload.addEventListener("progress", (e) => {
if (e.lengthComputable) {
// Accumulate progress across chunks
file.progress = chunk_byte_offset + e.loaded;
file.total = file.file_obj.size;
}
});
xhr.upload.addEventListener("load", () => {
if (chunk_index === total_chunks - 1) {
file.uploading = false;
}
});
xhr.addEventListener("error", () => {
file.failed = true;
reject();
});
xhr.onreadystatechange = () => {
if (xhr.readyState !== XMLHttpRequest.DONE) return;
if (xhr.status === 200) {
resolve();
file.request_succeeded = true;
let r = null;
let file_doc = null;
try {
r = JSON.parse(xhr.responseText);
if (r.message.doctype === "File") {
file_doc = r.message;
// Only the last chunk returns a meaningful response
if (chunk_index === total_chunks - 1) {
file.request_succeeded = true;
let r = null;
let file_doc = null;
try {
r = JSON.parse(xhr.responseText);
if (r.message?.doctype === "File") {
file_doc = r.message;
}
} catch (e) {
r = xhr.responseText;
}
} catch (e) {
r = xhr.responseText;
}
file.doc = file_doc;
if (props.on_success) {
props.on_success(file_doc, r);
}
if (
i == files.value.length - 1 &&
files.value.every((file) => file.request_succeeded)
) {
close_dialog.value = true;
}
if (show_web_link.value && file.file_url) {
close_dialog.value = true;
file.doc = file_doc;
if (props.on_success) {
props.on_success(file_doc, r);
}
if (
i == files.value.length - 1 &&
files.value.every((f) => f.request_succeeded)
) {
close_dialog.value = true;
}
}
resolve();
} else if (xhr.status === 403) {
reject();
file.failed = true;
let response = parse_error_response(xhr.responseText);
file.error_message = __("Not permitted. {0}.", [response.error_message || ""]);
if (response.server_messages.length) {
file.error_message += `\n${response.server_messages.join("\n")}`;
}
} else if (xhr.status === 413) {
reject();
} else if (xhr.status === 413) {
file.failed = true;
file.error_message = __("Size exceeds the maximum allowed file size.");
} else if (xhr.status === 417) {
reject();
// regular frappe.throw() in backend
} else if (xhr.status === 417) {
file.failed = true;
let response = parse_error_response(xhr.responseText);
file.error_message = response.server_messages.length
? response.server_messages.join("\n")
: __("File upload failed.");
if (show_web_link.value && web_link.value && file.file_url) {
web_link.value.invalid_input(__(file.error_message));
} else if (!files.value.includes(file)) {
@ -650,8 +657,8 @@ function upload_file(file, i) {
indicator: "red",
});
}
} else {
reject();
} else {
file.failed = true;
let detail =
xhr.statusText ||
@ -660,7 +667,6 @@ function upload_file(file, i) {
xhr.status === 0
? __("XMLHttpRequest Error")
: `${xhr.status} : ${detail}`;
let error = null;
try {
error = JSON.parse(xhr.responseText);
@ -668,61 +674,60 @@ function upload_file(file, i) {
// pass
}
frappe.request.cleanup({}, error);
reject();
}
};
xhr.open("POST", "/api/method/upload_file", true);
xhr.setRequestHeader("Accept", "application/json");
xhr.setRequestHeader("X-Frappe-CSRF-Token", frappe.csrf_token);
let form_data = new FormData();
if (chunk_blob) {
form_data.append("file", chunk_blob, file.name);
}
};
xhr.open("POST", "/api/method/upload_file", true);
xhr.setRequestHeader("Accept", "application/json");
xhr.setRequestHeader("X-Frappe-CSRF-Token", frappe.csrf_token);
let form_data = new FormData();
if (file.file_obj) {
form_data.append("file", file.file_obj, file.name);
}
form_data.append("is_private", +file.private);
form_data.append("folder", props.folder);
form_data.append("is_private", +file.private);
form_data.append("folder", props.folder);
form_data.append("total_file_size", file.file_obj?.size ?? 0);
if (file.file_url) {
form_data.append("file_url", file.file_url);
}
if (file.file_size) {
form_data.append("file_size", file.file_size);
}
if (file.file_name) {
form_data.append("file_name", file.file_name);
}
if (file.library_file_name) {
form_data.append("library_file_name", file.library_file_name);
}
if (use_chunks) {
form_data.append("chunk_index", chunk_index);
form_data.append("total_chunk_count", total_chunks);
form_data.append("chunk_byte_offset", chunk_byte_offset);
}
if (props.doctype) {
form_data.append("doctype", props.doctype);
}
if (file.file_url) form_data.append("file_url", file.file_url);
if (file.file_size) form_data.append("file_size", file.file_size);
if (file.file_name) form_data.append("file_name", file.file_name);
if (file.library_file_name)
form_data.append("library_file_name", file.library_file_name);
if (props.doctype) form_data.append("doctype", props.doctype);
if (props.docname) form_data.append("docname", props.docname);
if (props.fieldname) form_data.append("fieldname", props.fieldname);
if (props.method) form_data.append("method", props.method);
if (file.optimize) form_data.append("optimize", true);
if (props.attach_doc_image) {
form_data.append("max_width", 200);
form_data.append("max_height", 200);
}
if (props.docname) {
form_data.append("docname", props.docname);
}
xhr.send(form_data);
});
};
if (props.fieldname) {
form_data.append("fieldname", props.fieldname);
}
if (props.method) {
form_data.append("method", props.method);
}
if (file.optimize) {
form_data.append("optimize", true);
}
if (props.attach_doc_image) {
form_data.append("max_width", 200);
form_data.append("max_height", 200);
}
xhr.send(form_data);
});
// Slice and send chunks sequentially
let chunk_byte_offset = 0;
for (let chunk_index = 0; chunk_index < total_chunks; chunk_index++) {
const chunk_blob = file.file_obj
? file.file_obj.slice(chunk_byte_offset, chunk_byte_offset + CHUNK_SIZE)
: null;
await send_chunk(chunk_blob, chunk_index, chunk_byte_offset);
chunk_byte_offset += CHUNK_SIZE;
}
}
function parse_error_response(response_text) {
let error_message = "";
let server_messages = [];