diff --git a/.eslintrc b/.eslintrc index f94193e00e..c57be1d190 100644 --- a/.eslintrc +++ b/.eslintrc @@ -119,6 +119,7 @@ "get_url_arg": true, "QUnit": true, "Snap": true, - "mina": true + "mina": true, + "SocketIOFileClient" } } diff --git a/frappe/boot.py b/frappe/boot.py index 5d042f82ce..9f0c05a6ce 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -30,6 +30,7 @@ def get_bootinfo(): get_user(bootinfo) # system info + bootinfo.sitename = frappe.local.site bootinfo.sysdefaults = frappe.defaults.get_defaults() bootinfo.user_permissions = get_user_permissions() bootinfo.server_date = frappe.utils.nowdate() diff --git a/frappe/public/build.json b/frappe/public/build.json index 913adfe735..adeb8500db 100755 --- a/frappe/public/build.json +++ b/frappe/public/build.json @@ -126,6 +126,7 @@ "public/js/lib/moment/moment-with-locales.min.js", "public/js/lib/moment/moment-timezone-with-data.min.js", "public/js/lib/socket.io.min.js", + "public/js/lib/socket.io-file-client.js", "public/js/lib/markdown.js", "public/js/lib/jSignature.min.js", "public/js/frappe/translate.js", diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 68c63dae10..0610e6b551 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -29,7 +29,7 @@ frappe.Application = Class.extend({ this.startup(); }, startup: function() { - frappe.socket.init(); + frappe.socketio.init(); frappe.model.init(); if(frappe.boot.status==='failed') { diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js index 19b622e8ee..bde19db1f3 100644 --- a/frappe/public/js/frappe/request.js +++ b/frappe/public/js/frappe/request.js @@ -34,7 +34,7 @@ frappe.call = function(opts) { var callback = function(data, response_text) { if(data.task_id) { // async call, subscribe - frappe.socket.subscribe(data.task_id, opts); + frappe.socketio.subscribe(data.task_id, opts); if(opts.queued) { opts.queued(data); diff --git a/frappe/public/js/frappe/socketio_client.js b/frappe/public/js/frappe/socketio_client.js index c8af253bed..5427a129d8 100644 --- a/frappe/public/js/frappe/socketio_client.js +++ b/frappe/public/js/frappe/socketio_client.js @@ -1,4 +1,4 @@ -frappe.socket = { +frappe.socketio = { open_tasks: {}, open_docs: [], emit_queue: [], @@ -7,40 +7,40 @@ frappe.socket = { return; } - if (frappe.socket.socket) { + if (frappe.socketio.socket) { return; } if (frappe.boot.developer_mode) { // File watchers for development - frappe.socket.setup_file_watchers(); + frappe.socketio.setup_file_watchers(); } //Enable secure option when using HTTPS if (window.location.protocol == "https:") { - frappe.socket.socket = io.connect(frappe.socket.get_host(), {secure: true}); + frappe.socketio.socket = io.connect(frappe.socketio.get_host(), {secure: true}); } else if (window.location.protocol == "http:") { - frappe.socket.socket = io.connect(frappe.socket.get_host()); + frappe.socketio.socket = io.connect(frappe.socketio.get_host()); } else if (window.location.protocol == "file:") { - frappe.socket.socket = io.connect(window.localStorage.server); + frappe.socketio.socket = io.connect(window.localStorage.server); } - if (!frappe.socket.socket) { - console.log("Unable to connect to " + frappe.socket.get_host()); + if (!frappe.socketio.socket) { + console.log("Unable to connect to " + frappe.socketio.get_host()); return; } - frappe.socket.socket.on('msgprint', function(message) { + frappe.socketio.socket.on('msgprint', function(message) { frappe.msgprint(message); }); - frappe.socket.socket.on('eval_js', function(message) { + frappe.socketio.socket.on('eval_js', function(message) { eval(message); }); - frappe.socket.socket.on('progress', function(data) { + frappe.socketio.socket.on('progress', function(data) { if(data.progress) { data.percent = flt(data.progress[0]) / data.progress[1] * 100; } @@ -53,23 +53,24 @@ frappe.socket = { } }); - frappe.socket.setup_listeners(); - frappe.socket.setup_reconnect(); + frappe.socketio.setup_listeners(); + frappe.socketio.setup_reconnect(); + frappe.socketio.setup_fileupload(); $(document).on('form-load form-rename', function(e, frm) { if (frm.is_new()) { return; } - for (var i=0, l=frappe.socket.open_docs.length; i${filename} changed Click to Reload @@ -239,7 +246,7 @@ frappe.socket = { } // success - var opts = frappe.socket.open_tasks[data.task_id]; + var opts = frappe.socketio.open_tasks[data.task_id]; if(opts[method]) { opts[method](data); } @@ -264,15 +271,15 @@ frappe.socket = { frappe.provide("frappe.realtime"); frappe.realtime.on = function(event, callback) { - frappe.socket.socket && frappe.socket.socket.on(event, callback); + frappe.socketio.socket && frappe.socketio.socket.on(event, callback); }; frappe.realtime.off = function(event, callback) { - frappe.socket.socket && frappe.socket.socket.off(event, callback); + frappe.socketio.socket && frappe.socketio.socket.off(event, callback); } frappe.realtime.publish = function(event, message) { - if(frappe.socket.socket) { - frappe.socket.socket.emit(event, message); + if(frappe.socketio.socket) { + frappe.socketio.socket.emit(event, message); } } diff --git a/frappe/public/js/lib/socket.io-file-client.js b/frappe/public/js/lib/socket.io-file-client.js new file mode 100644 index 0000000000..351981dda1 --- /dev/null +++ b/frappe/public/js/lib/socket.io-file-client.js @@ -0,0 +1,259 @@ +"use strict"; +(function() { + + var instanceId = 0; + function getInstanceId() { + return instanceId++; + } + // note that this function invoked from call/apply, which has "this" binded + function _upload(file, options) { + options = options || {}; + + var self = this; + var socket = this.socket; + var chunkSize = this.chunkSize; + var transmissionDelay = this.transmissionDelay; + var uploadId = file.uploadId; + var uploadTo = options.uploadTo || ''; + var fileInfo = { + id: uploadId, + name: file.name, + size: file.size, + chunkSize: chunkSize, + sent: 0 + }; + + uploadTo && (fileInfo.uploadTo = uploadTo); + + // read file + var fileReader = new FileReader(); + fileReader.onloadend = function() { + var buffer = fileReader.result; + + // check file mime type if exists + if(self.accepts && self.accepts.length > 0) { + var found = false; + + for(var i = 0; i < self.accepts.length; i++) { + var accept = self.accepts[i]; + + if(file.type === accept) { + found = true; + break; + } + } + + if(!found) { + return self.emit('error', new Error('Not Acceptable file type ' + file.type + ' of ' + file.name + '. Type must be one of these: ' + self.accepts.join(', '))); + } + } + + // check file size + if(self.maxFileSize && self.maxFileSize > 0) { + if(file.size > +self.maxFileSize) { + return self.emit('error', new Error('Max Uploading File size must be under ' + self.maxFileSize + ' byte(s).')); + } + } + + // put into uploadingFiles list + self.uploadingFiles[uploadId] = fileInfo; + + // request the server to make a file + self.emit('start', { + name: fileInfo.name, + size: fileInfo.size, + uploadTo: uploadTo + }); + socket.emit('socket.io-file::createFile', fileInfo); + + function sendChunk() { + if(fileInfo.aborted) { + return; + } + + if(fileInfo.sent >= buffer.byteLength) { + socket.emit('socket.io-file::done::' + uploadId); + return; + } + + var chunk = buffer.slice(fileInfo.sent, fileInfo.sent + chunkSize); + + self.emit('stream', { + name: fileInfo.name, + size: fileInfo.size, + sent: fileInfo.sent, + uploadTo: uploadTo + }); + socket.once('socket.io-file::request::' + uploadId, sendChunk); + socket.emit('socket.io-file::stream::' + uploadId, chunk); + + fileInfo.sent += chunk.byteLength; + self.uploadingFiles[uploadId] = fileInfo; + } + socket.once('socket.io-file::request::' + uploadId, sendChunk); + socket.on('socket.io-file::complete::' + uploadId, function(info) { + self.emit('complete', info); + + socket.removeAllListeners('socket.io-file::abort::' + uploadId); + socket.removeAllListeners('socket.io-file::error::' + uploadId); + socket.removeAllListeners('socket.io-file::complete::' + uploadId); + + // remove from uploadingFiles list + delete self.uploadingFiles[uploadId]; + }); + socket.on('socket.io-file::abort::' + uploadId, function(info) { + fileInfo.aborted = true; + self.emit('abort', { + name: fileInfo.name, + size: fileInfo.size, + sent: fileInfo.sent, + wrote: info.wrote, + uploadTo: uploadTo + }); + }); + socket.on('socket.io-file::error::' + uploadId, function(err) { + self.emit('error', new Error(err.message)); + }); + }; + fileReader.readAsArrayBuffer(file); + } + + function SocketIOFileClient(socket, options) { + if(!socket) { + return this.emit('error', new Error('SocketIOFile requires Socket.')); + } + + this.instanceId = getInstanceId(); // using for identifying multiple file upload from SocketIOFileClient objects + this.uploadId = 0; // using for identifying each uploading + this.ev = {}; // event handlers + this.options = options || {}; + this.accepts = []; + this.maxFileSize = undefined; + this.socket = socket; + this.uploadingFiles = {}; + + var self = this; + + socket.once('socket.io-file::recvSync', function(settings) { + self.maxFileSize = settings.maxFileSize || undefined; + self.accepts = settings.accepts || []; + self.chunkSize = settings.chunkSize || 10240; + self.transmissionDelay = settings.transmissionDelay || 0; + + self.emit('ready'); + }); + socket.emit('socket.io-file::reqSync'); + } + SocketIOFileClient.prototype.getUploadId = function() { + return 'u_' + this.uploadId++; + } + SocketIOFileClient.prototype.upload = function(fileEl, options) { + if(!fileEl || + (fileEl.files && fileEl.files.length <= 0) || + fileEl.length <= 0 + ) { + this.emit('error', new Error('No file(s) to upload.')); + return []; + } + + var self = this; + var uploadIds = []; + + var files = fileEl.files ? fileEl.files : fileEl; + var loaded = 0; + + for(var i = 0; i < files.length; i++) { + var file = files[i]; + var uploadId = this.getUploadId(); + uploadIds.push(uploadId); + + file.uploadId = uploadId; + + _upload.call(self, file, options); + } + + return uploadIds; + }; + SocketIOFileClient.prototype.on = function(evName, fn) { + if(!this.ev[evName]) { + this.ev[evName] = []; + } + + this.ev[evName].push(fn); + return this; + }; + SocketIOFileClient.prototype.off = function(evName, fn) { + if(typeof evName === 'undefined') { + this.ev = []; + } + else if(typeof fn === 'undefined') { + if(this.ev[evName]) { + delete this.ev[evName]; + } + } + else { + var evList = this.ev[evName] || []; + + for(var i = 0; i < evList.length; i++) { + if(evList[i] === fn) { + evList = evList.splice(i, 1); + break; + } + } + } + + return this; + }; + SocketIOFileClient.prototype.emit = function(evName, args) { + var evList = this.ev[evName] || []; + + for(var i = 0; i < evList.length; i++) { + evList[i](args); + } + + return this; + }; + SocketIOFileClient.prototype.abort = function(id) { + var socket = this.socket; + socket.emit('socket.io-file::abort::' + id); + }; + SocketIOFileClient.prototype.destroy = function() { + var uploadingFiles = this.uploadingFiles; + + for(var key in uploadingFiles) { + this.abort(key); + } + + this.socket = null; + this.uploadingFiles = null; + this.ev = null; + }; + SocketIOFileClient.prototype.getUploadInfo = function() { + return JSON.parse(JSON.stringify(this.uploadingFiles)); + }; + + // module export + // CommonJS + if (typeof exports === "object" && typeof module !== "undefined") { + module.exports = SocketIOFileClient; + } + // RequireJS + else if (typeof define === "function" && define.amd) { + define(['SocketIOFileClient'], SocketIOFileClient); + } + else { + var g; + + if (typeof window !== "undefined") { + g = window; + } + else if (typeof global !== "undefined") { + g = global; + } + else if (typeof self !== "undefined") { + g = self; + } + + g.SocketIOFileClient = SocketIOFileClient; + } +})(); \ No newline at end of file diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index 2d32215d6a..691bc166e4 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -15,6 +15,7 @@ from six import text_type class MaxFileSizeReachedError(frappe.ValidationError): pass + def get_file_url(file_data_name): data = frappe.db.get_value("File", file_data_name, ["file_name", "file_url"], as_dict=True) return data.file_url or data.file_name @@ -97,55 +98,6 @@ def get_uploaded_content(): frappe.msgprint(_('No file attached')) return None, None -def extract_images_from_doc(doc, fieldname): - content = doc.get(fieldname) - content = extract_images_from_html(doc, content) - if frappe.flags.has_dataurl: - doc.set(fieldname, content) - -def extract_images_from_html(doc, content): - frappe.flags.has_dataurl = False - - def _save_file(match): - data = match.group(1) - data = data.split("data:")[1] - headers, content = data.split(",") - - if "filename=" in headers: - filename = headers.split("filename=")[-1] - - # decode filename - if not isinstance(filename, text_type): - filename = text_type(filename, 'utf-8') - else: - mtype = headers.split(";")[0] - filename = get_random_filename(content_type=mtype) - - doctype = doc.parenttype if doc.parent else doc.doctype - name = doc.parent or doc.name - - # TODO fix this - file_url = save_file(filename, content, doctype, name, decode=True).get("file_url") - if not frappe.flags.has_dataurl: - frappe.flags.has_dataurl = True - - return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) - - return content - -def get_random_filename(extn=None, content_type=None): - if extn: - if not extn.startswith("."): - extn = "." + extn - - elif content_type: - extn = mimetypes.guess_extension(content_type) - - return random_string(7) + (extn or "") - def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0): if decode: if isinstance(content, text_type): @@ -370,3 +322,52 @@ def download_file(file_url): frappe.local.response.filename = file_url[file_url.rfind("/")+1:] frappe.local.response.filecontent = filedata frappe.local.response.type = "download" + +def extract_images_from_doc(doc, fieldname): + content = doc.get(fieldname) + content = extract_images_from_html(doc, content) + if frappe.flags.has_dataurl: + doc.set(fieldname, content) + +def extract_images_from_html(doc, content): + frappe.flags.has_dataurl = False + + def _save_file(match): + data = match.group(1) + data = data.split("data:")[1] + headers, content = data.split(",") + + if "filename=" in headers: + filename = headers.split("filename=")[-1] + + # decode filename + if not isinstance(filename, text_type): + filename = text_type(filename, 'utf-8') + else: + mtype = headers.split(";")[0] + filename = get_random_filename(content_type=mtype) + + doctype = doc.parenttype if doc.parent else doc.doctype + name = doc.parent or doc.name + + # TODO fix this + file_url = save_file(filename, content, doctype, name, decode=True).get("file_url") + if not frappe.flags.has_dataurl: + frappe.flags.has_dataurl = True + + return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + + return content + +def get_random_filename(extn=None, content_type=None): + if extn: + if not extn.startswith("."): + extn = "." + extn + + elif content_type: + extn = mimetypes.guess_extension(content_type) + + return random_string(7) + (extn or "") diff --git a/socketio.js b/socketio.js index cd3eb64aa8..98cdf16a78 100644 --- a/socketio.js +++ b/socketio.js @@ -134,6 +134,34 @@ io.on('connection', function(socket){ }); }); + var uploader = new SocketIOFile(socket, { + // uploadDir: { // multiple directories + // music: 'data/music', + // document: 'data/document' + // }, + uploadDir: 'sites/uploads', + // maxFileSize: 4194304, // 4 MB. default is undefined(no limit) + chunkSize: 10240, // default is 10240(1KB) + overwrite: true // overwrite file if exists, default is true. + }); + uploader.on('start', (fileInfo) => { + console.log('Start uploading'); + console.log(fileInfo); + }); + uploader.on('stream', (fileInfo) => { + console.log(`${fileInfo.wrote} / ${fileInfo.size} byte(s)`); + }); + uploader.on('complete', (fileInfo) => { + console.log('Upload Complete.'); + console.log(fileInfo); + }); + uploader.on('error', (err) => { + console.log('Error!', err); + }); + uploader.on('abort', (fileInfo) => { + console.log('Aborted: ', fileInfo); + }); + // socket.on('disconnect', function (arguments) { // console.log("user disconnected", arguments); // });