Merge branch 'master' into develop

.eslintrc

@@ -5,7 +5,7 @@
 		"es6": true
 	},
 	"parserOptions": {
-		"ecmaVersion": 6,
+		"ecmaVersion": 8,
 		"sourceType": "module"
 	},
 	"extends": "eslint:recommended",

frappe/__init__.py

@@ -24,7 +24,7 @@ if sys.version[0] == '2':
 	reload(sys)
 	sys.setdefaultencoding("utf-8")
 
-__version__ = '11.1.13'
+__version__ = '11.1.14'
 __title__ = "Frappe Framework"
 
 local = Local()

frappe/chat/doctype/chat_profile/chat_profile.py

@@ -48,29 +48,31 @@ def authenticate(user):
 @frappe.whitelist()
 def get(user, fields = None):
 	duser   = frappe.get_doc('User', user)
-	dprof   = frappe.get_doc('Chat Profile', user)
 
-	# If you're adding something here, make sure the client recieves it.
-	profile = dict(
-		# User
-		name       = duser.name,
-		email      = duser.email,
-		first_name = duser.first_name,
-		last_name  = duser.last_name,
-		username   = duser.username,
-		avatar     = duser.user_image,
-		bio        = duser.bio,
-		# Chat Profile
-		status             = dprof.status,
-		chat_background    = dprof.chat_background,
-		message_preview    = bool(dprof.message_preview),
-		notification_tones = bool(dprof.notification_tones),
-		conversation_tones = bool(dprof.conversation_tones),
-		enable_chat        = bool(dprof.enable_chat)
-	)
-	profile = filter_dict(profile, fields)
+	if frappe.db.exists('Chat Profile', user):
+		dprof = frappe.get_doc('Chat Profile', user)
 
-	return dictify(profile)
+		# If you're adding something here, make sure the client recieves it.
+		profile = dict(
+			# User
+			name       = duser.name,
+			email      = duser.email,
+			first_name = duser.first_name,
+			last_name  = duser.last_name,
+			username   = duser.username,
+			avatar     = duser.user_image,
+			bio        = duser.bio,
+			# Chat Profile
+			status             = dprof.status,
+			chat_background    = dprof.chat_background,
+			message_preview    = bool(dprof.message_preview),
+			notification_tones = bool(dprof.notification_tones),
+			conversation_tones = bool(dprof.conversation_tones),
+			enable_chat        = bool(dprof.enable_chat)
+		)
+		profile = filter_dict(profile, fields)
+
+		return dictify(profile)
 
 @frappe.whitelist()
 def create(user, exists_ok = False, fields = None):

frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py

@@ -66,7 +66,9 @@ class RolePermissionforPageandReport(Document):
 
 	def update_disable_prepared_report(self):
 		if self.report:
-			frappe.db.set_value('Report', self.report, 'disable_prepared_report', self.disable_prepared_report)
+			# intentionally written update query in frappe.db.sql instead of frappe.db.set_value
+			frappe.db.sql(""" update `tabReport` set disable_prepared_report = %s
+				where name = %s""", (self.disable_prepared_report, self.report))
 
 	def get_args(self, row=None):
 		name = self.page if self.set_role_for == 'Page' else self.report
@@ -75,7 +77,7 @@ class RolePermissionforPageandReport(Document):
 		return {
 			check_for_field: name
 		}
-		
+
 	def get_roles(self):
 		roles = []
 		for data in self.roles:

frappe/core/doctype/user/user.py

@@ -1048,7 +1048,7 @@ def update_roles(role_profile):
 		user.set('roles', [])
 		user.add_roles(*roles)
 
-def create_contact(user, ignore_links=False):
+def create_contact(user, ignore_links=False, ignore_mandatory=False):
 	if user.name in ["Administrator", "Guest"]: return
 
 	if not frappe.db.get_value("Contact", {"email_id": user.email}):
@@ -1061,7 +1061,7 @@ def create_contact(user, ignore_links=False):
 			"gender": user.gender,
 			"phone": user.phone,
 			"mobile_no": user.mobile_no
-		}).insert(ignore_permissions=True, ignore_links=ignore_links)
+		}).insert(ignore_permissions=True, ignore_links=ignore_links, ignore_mandatory=ignore_mandatory)
 
 
 @frappe.whitelist()

frappe/custom/doctype/customize_form/customize_form.py

@@ -38,6 +38,7 @@ docfield_properties = {
 	'fieldtype': 'Select',
 	'options': 'Text',
 	'fetch_from': 'Small Text',
+	'fetch_if_empty': 'Check',
 	'permlevel': 'Int',
 	'width': 'Data',
 	'print_width': 'Data',

frappe/desk/doctype/todo/test_todo.py

@@ -46,3 +46,31 @@ class TestToDo(unittest.TestCase):
 
 		self.assertEqual(todo.assigned_by_full_name,
 			frappe.db.get_value('User', todo.assigned_by, 'full_name'))
+
+def test_fetch_if_empty(self):
+		frappe.db.sql('delete from tabToDo')
+
+		# Allow user changes
+		todo_meta = frappe.get_doc('DocType', 'ToDo')
+		field = todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0]
+		field.fetch_from = 'assigned_by.full_name'
+		field.fetch_if_empty = 1
+		todo_meta.save()
+
+		frappe.clear_cache(doctype='ToDo')
+
+		todo = frappe.get_doc(dict(doctype='ToDo', description='test todo',
+			assigned_by='Administrator', assigned_by_full_name='Admin')).insert()
+
+		self.assertEqual(todo.assigned_by_full_name, 'Admin')
+
+		# Overwrite user changes
+		todo_meta = frappe.get_doc('DocType', 'ToDo')
+		todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_if_empty = 0
+		todo_meta.save()
+
+		todo.reload()
+		todo.save()
+
+		self.assertEqual(todo.assigned_by_full_name,
+			frappe.db.get_value('User', todo.assigned_by, 'full_name'))

frappe/model/base_document.py

@@ -459,9 +459,12 @@ class BaseDocument(object):
 				# that are mapped as link_fieldname.source_fieldname in Options of
 				# Readonly or Data or Text type fields
 
-				# NOTE: All fields will be replaced, if you want manual changes to stay
-				# use `frm.add_fetch`
-				fields_to_fetch = self.meta.get_fields_to_fetch(df.fieldname)
+				fields_to_fetch = [
+					_df for _df in self.meta.get_fields_to_fetch(df.fieldname)
+					if
+						not _df.get('fetch_if_empty')
+						or (_df.get('fetch_if_empty') and not self.get(_df.fieldname))
+				]
 
 				if not fields_to_fetch:
 					# cache a single value type

frappe/model/db_query.py

@@ -173,6 +173,9 @@ class DatabaseQuery(object):
 				except ValueError:
 					self.fields = [f.strip() for f in self.fields.split(",")]
 
+		# remove empty strings / nulls in fields
+		self.fields = [f for f in self.fields if f]
+
 		for filter_name in ["filters", "or_filters"]:
 			filters = getattr(self, filter_name)
 			if isinstance(filters, string_types):

frappe/model/document.py

@@ -1217,7 +1217,7 @@ class Document(BaseDocument):
 
 		if file_lock.lock_exists(self.get_signature()):
 			frappe.throw(_('This document is currently queued for execution. Please try again'),
-				title=_('Document Queued'), indicator='red')
+				title=_('Document Queued'))
 
 		self.lock()
 		enqueue('frappe.model.document.execute_action', doctype=self.doctype, name=self.name,

frappe/model/meta.py

@@ -232,7 +232,7 @@ class Meta(Document):
 		are to be fetched and updated for a particular link field
 
 		These fields are of type Data, Link, Text, Readonly and their
-		options property is set as `link_fieldname`.`source_fieldname`'''
+		fetch_from property is set as `link_fieldname`.`source_fieldname`'''
 
 		out = []
 

frappe/patches/v11_0/create_contact_for_user.py

@@ -13,4 +13,4 @@ def execute():
 			user.first_name = re.sub("[<>]+", '', frappe.safe_decode(user.first_name))
 		if user.last_name:
 			user.last_name  = re.sub("[<>]+", '', frappe.safe_decode(user.last_name))
-		create_contact(user, ignore_links=True)
+		create_contact(user, ignore_links=True, ignore_mandatory=True)

frappe/public/js/frappe/form/controls/text_editor.js

@@ -20,6 +20,25 @@ Table.create = (value) => {
 }
 Quill.register(Table, true);
 
+// link without href
+var Link = Quill.import('formats/link');
+
+class MyLink extends Link {
+	static create(value) {
+		let node = super.create(value);
+		value = this.sanitize(value);
+		node.setAttribute('href', value);
+		if(value.startsWith('/') || value.indexOf(window.location.host)) {
+			// no href if internal link
+			node.removeAttribute('target');
+		}
+		return node;
+	}
+}
+
+Quill.register(MyLink);
+
+
 // hidden blot
 class HiddenBlock extends Block {
 	static create(value) {
@@ -44,13 +63,11 @@ Uploader.DEFAULTS.mimetypes.push('image/gif');
 // inline style
 const BackgroundStyle = Quill.import('attributors/style/background');
 const ColorStyle = Quill.import('attributors/style/color');
-const SizeStyle = Quill.import('attributors/style/size');
 const FontStyle = Quill.import('attributors/style/font');
 const AlignStyle = Quill.import('attributors/style/align');
 const DirectionStyle = Quill.import('attributors/style/direction');
 Quill.register(BackgroundStyle, true);
 Quill.register(ColorStyle, true);
-Quill.register(SizeStyle, true);
 Quill.register(FontStyle, true);
 Quill.register(AlignStyle, true);
 Quill.register(DirectionStyle, true);
@@ -140,6 +157,7 @@ frappe.ui.form.ControlTextEditor = frappe.ui.form.ControlCode.extend({
 		return [
 			[{ 'header': [1, 2, 3, false] }],
 			['bold', 'italic', 'underline'],
+			[{ 'color': [] }, { 'background': [] }],
 			['blockquote', 'code-block'],
 			['link', 'image'],
 			[{ 'list': 'ordered' }, { 'list': 'bullet' }],

frappe/public/js/frappe/form/footer/timeline.js

@@ -308,7 +308,7 @@ frappe.ui.form.Timeline = class Timeline {
 	}
 
 	prepare_timeline_item(c) {
-		if(!c.sender) c.sender = c.owner;
+		if(!c.sender) c.sender = c.owner || 'Guest';
 
 		if(c.sender && c.sender.indexOf("<")!==-1) {
 			c.sender = c.sender.split("<")[1].split(">")[0];

frappe/public/js/frappe/model/create_new.js

@@ -90,7 +90,7 @@ $.extend(frappe.model, {
 		for(var fid=0;fid<docfields.length;fid++) {
 			var f = docfields[fid];
 			if(!in_list(frappe.model.no_value_type, f.fieldtype) && doc[f.fieldname]==null) {
-				var v = frappe.model.get_default_value(f, doc, parent_doc);
+				var v = !f.depends_on || doc[f.depends_on] ? frappe.model.get_default_value(f, doc, parent_doc) : null;
 				if(v) {
 					if(in_list(["Int", "Check"], f.fieldtype))
 						v = cint(v);

frappe/public/js/frappe/ui/tags.js

@@ -77,13 +77,9 @@ frappe.ui.Tags = class {
 	}
 
 	removeTag(label) {
+		label = frappe.utils.xss_sanitise(label);
 		if(this.tagsList.includes(label)) {
-			let $tag = this.$ul.find(`.frappe-tag[data-tag-label="${label}"]`);
-
-			// Just don't remove tag, but also the li DOM.
-			$tag.parent('.tags-list-item').remove();
 			this.tagsList.splice(this.tagsList.indexOf(label), 1);
-
 			this.onTagRemove && this.onTagRemove(label);
 		}
 	}
@@ -119,6 +115,7 @@ frappe.ui.Tags = class {
 
 		$removeTag.on("click", () => {
 			this.removeTag($removeTag.attr('data-tag-label'));
+			$removeTag.closest('.tags-list-item').remove();
 		});
 
 		if(this.onTagClick) {

frappe/public/js/frappe/views/reports/query_report.js

@@ -52,10 +52,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		this.secondary_action = {
 			label: __('Refresh'),
 			action: () => {
-				if(this.execution_time > 2) {
-					this.setup_progress_bar();
-				}
-
+				this.setup_progress_bar();
 				this.refresh();
 			}
 		};
@@ -171,8 +168,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 
 	setup_progress_bar() {
 		let seconds_elapsed = 0;
-		const execution_time = this.report_settings.execution_time < 10
-			? 10 : this.report_settings.execution_time;
+		const execution_time = this.report_settings.execution_time || 0;
+
+		if (execution_time < 5) return;
 
 		this.interval = setInterval(function()  {
 			seconds_elapsed += 1;

frappe/public/less/quill.less

@@ -9,6 +9,9 @@
 }
 
 .ql-editor {
+    font-family: @font-stack;
+    line-height: 1.6;
+
 	h1, h2, h3, h4, h5 {
 		margin-top: 0.5em;
 		margin-bottom: 0.25em;

frappe/utils/file_manager.py

@@ -0,0 +1,453 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+import os, base64, re, json
+import hashlib
+import mimetypes
+import io
+from frappe.utils import get_hook_method, get_files_path, random_string, encode, cstr, call_hook_method, cint
+from frappe import _
+from frappe import conf
+from copy import copy
+from six.moves.urllib.parse import unquote
+from six import text_type, PY2, string_types
+
+
+class MaxFileSizeReachedError(frappe.ValidationError):
+	pass
+
+
+def get_file_url(file_data_name):
+	data = frappe.db.get_value("File", file_data_name, ["file_name", "file_url"], as_dict=True)
+	return data.file_url or data.file_name
+
+
+def upload():
+	# get record details
+	dt = frappe.form_dict.doctype
+	dn = frappe.form_dict.docname
+	file_url = frappe.form_dict.file_url
+	filename = frappe.form_dict.filename
+	frappe.form_dict.is_private = cint(frappe.form_dict.is_private)
+
+	if not filename and not file_url:
+		frappe.msgprint(_("Please select a file or url"),
+			raise_exception=True)
+
+	file_doc = get_file_doc()
+
+	comment = {}
+	if dt and dn:
+		comment = frappe.get_doc(dt, dn).add_comment("Attachment",
+			_("added {0}").format("<a href='{file_url}' target='_blank'>{file_name}</a>{icon}".format(**{
+				"icon": ' <i class="fa fa-lock text-warning"></i>' \
+					if file_doc.is_private else "",
+				"file_url": file_doc.file_url.replace("#", "%23") \
+					if file_doc.file_name else file_doc.file_url,
+				"file_name": file_doc.file_name or file_doc.file_url
+			})))
+
+	return {
+		"name": file_doc.name,
+		"file_name": file_doc.file_name,
+		"file_url": file_doc.file_url,
+		"is_private": file_doc.is_private,
+		"comment": comment.as_dict() if comment else {}
+	}
+
+def get_file_doc(dt=None, dn=None, folder=None, is_private=None, df=None):
+	'''returns File object (Document) from given parameters or form_dict'''
+	r = frappe.form_dict
+
+	if dt is None: dt = r.doctype
+	if dn is None: dn = r.docname
+	if df is None: df = r.docfield
+	if folder is None: folder = r.folder
+	if is_private is None: is_private = r.is_private
+
+	if r.filedata:
+		file_doc = save_uploaded(dt, dn, folder, is_private, df)
+
+	elif r.file_url:
+		file_doc = save_url(r.file_url, r.filename, dt, dn, folder, is_private, df)
+
+	return file_doc
+
+def save_uploaded(dt, dn, folder, is_private, df=None):
+	fname, content = get_uploaded_content()
+	if content:
+		return save_file(fname, content, dt, dn, folder, is_private=is_private, df=df);
+	else:
+		raise Exception
+
+def save_url(file_url, filename, dt, dn, folder, is_private, df=None):
+	# if not (file_url.startswith("http://") or file_url.startswith("https://")):
+	# 	frappe.msgprint("URL must start with 'http://' or 'https://'")
+	# 	return None, None
+
+	file_url = unquote(file_url)
+	file_size = frappe.form_dict.file_size
+
+	f = frappe.get_doc({
+		"doctype": "File",
+		"file_url": file_url,
+		"file_name": filename,
+		"attached_to_doctype": dt,
+		"attached_to_name": dn,
+		"attached_to_field": df,
+		"folder": folder,
+		"file_size": file_size,
+		"is_private": is_private
+	})
+	f.flags.ignore_permissions = True
+	try:
+		f.insert()
+	except frappe.DuplicateEntryError:
+		return frappe.get_doc("File", f.duplicate_entry)
+	return f
+
+
+def get_uploaded_content():
+	# should not be unicode when reading a file, hence using frappe.form
+	if 'filedata' in frappe.form_dict:
+		if "," in frappe.form_dict.filedata:
+			frappe.form_dict.filedata = frappe.form_dict.filedata.rsplit(",", 1)[1]
+		frappe.uploaded_content = base64.b64decode(frappe.form_dict.filedata)
+		frappe.uploaded_filename = frappe.form_dict.filename
+		return frappe.uploaded_filename, frappe.uploaded_content
+	else:
+		frappe.msgprint(_('No file attached'))
+		return None, None
+
+def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, df=None):
+	if decode:
+		if isinstance(content, text_type):
+			content = content.encode("utf-8")
+
+		if b"," in content:
+			content = content.split(b",")[1]
+		content = base64.b64decode(content)
+
+	file_size = check_max_file_size(content)
+	content_hash = get_content_hash(content)
+	content_type = mimetypes.guess_type(fname)[0]
+	fname = get_file_name(fname, content_hash[-6:])
+	file_data = get_file_data_from_hash(content_hash, is_private=is_private)
+	if not file_data:
+		call_hook_method("before_write_file", file_size=file_size)
+
+		write_file_method = get_hook_method('write_file', fallback=save_file_on_filesystem)
+		file_data = write_file_method(fname, content, content_type=content_type, is_private=is_private)
+		file_data = copy(file_data)
+
+	file_data.update({
+		"doctype": "File",
+		"attached_to_doctype": dt,
+		"attached_to_name": dn,
+		"attached_to_field": df,
+		"folder": folder,
+		"file_size": file_size,
+		"content_hash": content_hash,
+		"is_private": is_private
+	})
+
+	f = frappe.get_doc(file_data)
+	f.flags.ignore_permissions = True
+	try:
+		f.insert()
+	except frappe.DuplicateEntryError:
+		return frappe.get_doc("File", f.duplicate_entry)
+
+	return f
+
+
+def get_file_data_from_hash(content_hash, is_private=0):
+	for name in frappe.db.sql_list("select name from `tabFile` where content_hash=%s and is_private=%s", (content_hash, is_private)):
+		b = frappe.get_doc('File', name)
+		return {k: b.get(k) for k in frappe.get_hooks()['write_file_keys']}
+	return False
+
+
+def save_file_on_filesystem(fname, content, content_type=None, is_private=0):
+	fpath = write_file(content, fname, is_private)
+
+	if is_private:
+		file_url = "/private/files/{0}".format(fname)
+	else:
+		file_url = "/files/{0}".format(fname)
+
+	return {
+		'file_name': os.path.basename(fpath),
+		'file_url': file_url
+	}
+
+
+def get_max_file_size():
+	return conf.get('max_file_size') or 10485760
+
+
+def check_max_file_size(content):
+	max_file_size = get_max_file_size()
+	file_size = len(content)
+
+	if file_size > max_file_size:
+		frappe.msgprint(_("File size exceeded the maximum allowed size of {0} MB").format(
+			max_file_size / 1048576),
+			raise_exception=MaxFileSizeReachedError)
+
+	return file_size
+
+
+def write_file(content, fname, is_private=0):
+	"""write file to disk with a random name (to compare)"""
+	file_path = get_files_path(is_private=is_private)
+
+	# create directory (if not exists)
+	frappe.create_folder(file_path)
+	# write the file
+	if isinstance(content, text_type):
+		content = content.encode()
+	with open(os.path.join(file_path.encode('utf-8'), fname.encode('utf-8')), 'wb+') as f:
+		f.write(content)
+
+	return get_files_path(fname, is_private=is_private)
+
+
+def remove_all(dt, dn, from_delete=False):
+	"""remove all files in a transaction"""
+	try:
+		for fid in frappe.db.sql_list("""select name from `tabFile` where
+			attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)):
+			remove_file(fid, dt, dn, from_delete)
+	except Exception as e:
+		if e.args[0]!=1054: raise # (temp till for patched)
+
+
+def remove_file_by_url(file_url, doctype=None, name=None):
+	if doctype and name:
+		fid = frappe.db.get_value("File", {"file_url": file_url,
+			"attached_to_doctype": doctype, "attached_to_name": name})
+	else:
+		fid = frappe.db.get_value("File", {"file_url": file_url})
+
+	if fid:
+		return remove_file(fid)
+
+
+def remove_file(fid, attached_to_doctype=None, attached_to_name=None, from_delete=False):
+	"""Remove file and File entry"""
+	file_name = None
+	if not (attached_to_doctype and attached_to_name):
+		attached = frappe.db.get_value("File", fid,
+			["attached_to_doctype", "attached_to_name", "file_name"])
+		if attached:
+			attached_to_doctype, attached_to_name, file_name = attached
+
+	ignore_permissions, comment = False, None
+	if attached_to_doctype and attached_to_name and not from_delete:
+		doc = frappe.get_doc(attached_to_doctype, attached_to_name)
+		ignore_permissions = doc.has_permission("write") or False
+		if frappe.flags.in_web_form:
+			ignore_permissions = True
+		if not file_name:
+			file_name = frappe.db.get_value("File", fid, "file_name")
+		comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name))
+
+	frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions)
+
+	return comment
+
+
+def delete_file_data_content(doc, only_thumbnail=False):
+	method = get_hook_method('delete_file_data_content', fallback=delete_file_from_filesystem)
+	method(doc, only_thumbnail=only_thumbnail)
+
+
+def delete_file_from_filesystem(doc, only_thumbnail=False):
+	"""Delete file, thumbnail from File document"""
+	if only_thumbnail:
+		delete_file(doc.thumbnail_url)
+	else:
+		delete_file(doc.file_url)
+		delete_file(doc.thumbnail_url)
+
+
+def delete_file(path):
+	"""Delete file from `public folder`"""
+	if path:
+		if ".." in path.split("/"):
+			frappe.msgprint(_("It is risky to delete this file: {0}. Please contact your System Manager.").format(path))
+
+		parts = os.path.split(path.strip("/"))
+		if parts[0]=="files":
+			path = frappe.utils.get_site_path("public", "files", parts[-1])
+
+		else:
+			path = frappe.utils.get_site_path("private", "files", parts[-1])
+
+		path = encode(path)
+		if os.path.exists(path):
+			os.remove(path)
+
+
+def get_file(fname):
+	"""Returns [`file_name`, `content`] for given file name `fname`"""
+	file_path = get_file_path(fname)
+
+	# read the file
+	if PY2:
+		with open(encode(file_path)) as f:
+			content = f.read()
+	else:
+		with io.open(encode(file_path), mode='rb') as f:
+			content = f.read()
+			try:
+				# for plain text files
+				content = content.decode()
+			except UnicodeDecodeError:
+				# for .png, .jpg, etc
+				pass
+
+	return [file_path.rsplit("/", 1)[-1], content]
+
+
+def get_file_path(file_name):
+	"""Returns file path from given file name"""
+	f = frappe.db.sql("""select file_url from `tabFile`
+		where name=%s or file_name=%s""", (file_name, file_name))
+	if f:
+		file_name = f[0][0]
+
+	file_path = file_name
+
+	if "/" not in file_path:
+		file_path = "/files/" + file_path
+
+	if file_path.startswith("/private/files/"):
+		file_path = get_files_path(*file_path.split("/private/files/", 1)[1].split("/"), is_private=1)
+
+	elif file_path.startswith("/files/"):
+		file_path = get_files_path(*file_path.split("/files/", 1)[1].split("/"))
+
+	else:
+		frappe.throw(_("There is some problem with the file url: {0}").format(file_path))
+
+	return file_path
+
+
+def get_content_hash(content):
+	if isinstance(content, text_type):
+		content = content.encode()
+	return hashlib.md5(content).hexdigest()
+
+
+def get_file_name(fname, optional_suffix):
+	# convert to unicode
+	fname = cstr(fname)
+
+	n_records = frappe.db.sql("select name from `tabFile` where file_name=%s", fname)
+	if len(n_records) > 0 or os.path.exists(encode(get_files_path(fname))):
+		f = fname.rsplit('.', 1)
+		if len(f) == 1:
+			partial, extn = f[0], ""
+		else:
+			partial, extn = f[0], "." + f[1]
+		return '{partial}{suffix}{extn}'.format(partial=partial, extn=extn, suffix=optional_suffix)
+	return fname
+
+
+@frappe.whitelist()
+def download_file(file_url):
+	"""
+	Download file using token and REST API. Valid session or
+	token is required to download private files.
+
+	Method : GET
+	Endpoint : frappe.utils.file_manager.download_file
+	URL Params : file_name = /path/to/file relative to site path
+	"""
+	file_doc = frappe.get_doc("File", {"file_url":file_url})
+	file_doc.check_permission("read")
+	path = os.path.join(get_files_path(), os.path.basename(file_url))
+
+	with open(path, "rb") as fileobj:
+		filedata = fileobj.read()
+	frappe.local.response.filename = os.path.basename(file_url)
+	frappe.local.response.filecontent = filedata
+	frappe.local.response.type = "download"
+
+def extract_images_from_doc(doc, fieldname):
+	content = doc.get(fieldname)
+	content = extract_images_from_html(doc, content)
+	if frappe.flags.has_dataurl:
+		doc.set(fieldname, content)
+
+
+def extract_images_from_html(doc, content):
+	frappe.flags.has_dataurl = False
+
+	def _save_file(match):
+		data = match.group(1)
+		data = data.split("data:")[1]
+		headers, content = data.split(",")
+
+		if "filename=" in headers:
+			filename = headers.split("filename=")[-1]
+
+			# decode filename
+			if not isinstance(filename, text_type):
+				filename = text_type(filename, 'utf-8')
+		else:
+			mtype = headers.split(";")[0]
+			filename = get_random_filename(content_type=mtype)
+
+		doctype = doc.parenttype if doc.parent else doc.doctype
+		name = doc.parent or doc.name
+
+		# TODO fix this
+		file_url = save_file(filename, content, doctype, name, decode=True).get("file_url")
+		if not frappe.flags.has_dataurl:
+			frappe.flags.has_dataurl = True
+
+		return '<img src="{file_url}"'.format(file_url=file_url)
+
+	if content:
+		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
+
+	return content
+
+
+def get_random_filename(extn=None, content_type=None):
+	if extn:
+		if not extn.startswith("."):
+			extn = "." + extn
+
+	elif content_type:
+		extn = mimetypes.guess_extension(content_type)
+
+	return random_string(7) + (extn or "")
+
+@frappe.whitelist(allow_guest=True)
+def validate_filename(filename):
+	from frappe.utils import now_datetime
+	timestamp = now_datetime().strftime(" %Y-%m-%d %H:%M:%S")
+	fname = get_file_name(filename, timestamp)
+	return fname
+
+@frappe.whitelist()
+def add_attachments(doctype, name, attachments):
+	'''Add attachments to the given DocType'''
+	if isinstance(attachments, string_types):
+		attachments = json.loads(attachments)
+	# loop through attachments
+	files =[]
+	for a in attachments:
+		if isinstance(a, string_types):
+			attach = frappe.db.get_value("File", {"name":a}, ["file_name", "file_url", "is_private"], as_dict=1)
+			# save attachments to new doc
+			f = save_url(attach.file_url, attach.file_name, doctype, name, "Home/Attachments", attach.is_private)
+			files.append(f)
+
+	return files

frappe/utils/xlsxutils.py

@@ -61,10 +61,17 @@ def handle_html(data):
 	obj = HTML2Text()
 	obj.ignore_links = True
 	obj.body_width = 0
-	value = obj.handle(h)
+
+	try:
+		value = obj.handle(h)
+	except Exception:
+		# unable to parse html, send it raw
+		return value
+
 	value = ", ".join(value.split('  \n'))
 	value = " ".join(value.split('\n'))
 	value = ", ".join(value.split('# '))
+
 	return value
 
 def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=None):

frappe/website/doctype/web_form/templates/web_form.html

@@ -153,8 +153,8 @@
 				<!-- save/next button -->
 				{% if (loop.index == layout|len or frappe.form_dict.new) %}
 					{% if not read_only %}
-				    <button type="submit" class="btn btn-primary btn-sm btn-form-submit">
-				    	{{ _(button_label or "Save") }}</button>
+				    <button type="submit" class="btn btn-primary btn-sm btn-form-submit footer-button">
+				    	{{ _("Save") }}</button>
 					{% endif %}
 				{% elif layout|len > 1 %}
 				    <button class="btn btn-primary btn-sm btn-change-section"

frappe/website/js/web_form.js

@@ -36,6 +36,11 @@ frappe.ready(function() {
 	setTimeout(() => {
 		$('body').css('display', 'block');
 
+		// remove footer save button if form height is less than window height
+		if($('.webform-wrapper').height() < window.innerHeight) {
+			$(".footer-button").addClass("hide");
+		}
+
 		if (frappe.init_client_script) {
 			frappe.init_client_script();
 

frappe/website/js/website.js

@@ -13,35 +13,34 @@ $.extend(frappe, {
 		lang: 'en'
 	},
 	_assets_loaded: [],
-	require: function(url, callback) {
-
-		let async = false;
-		if (callback) {
-			async = true;
+	require: async function(links, callback) {
+		if (typeof (links) === 'string') {
+			links = [links];
 		}
-
-		if(frappe._assets_loaded.indexOf(url)!==-1) {
-			callback && callback();
-			return;
+		for (let link of links) {
+			await this.add_asset_to_head(link);
 		}
-
-		return $.ajax({
-			url: url,
-			async: async,
-			dataType: "text",
-			success: function(data) {
-				var el;
-				if(url.split(".").splice(-1) == "js") {
-					el = document.createElement('script');
-				} else {
-					el = document.createElement('style');
-				}
-				el.appendChild(document.createTextNode(data));
-				document.getElementsByTagName('head')[0].appendChild(el);
-				frappe._assets_loaded.push(url);
-
-				callback && callback();
+		callback && callback();
+	},
+	add_asset_to_head(link) {
+		return new Promise(resolve => {
+			if (frappe._assets_loaded.includes(link)) return resolve();
+			let el;
+			if(link.split('.').pop() === 'js') {
+				el = document.createElement('script');
+				el.type = 'text/javascript';
+				el.src = link;
+			} else {
+				el = document.createElement('link');
+				el.type = 'text/css';
+				el.rel = 'stylesheet';
+				el.href = link;
 			}
+			document.getElementsByTagName('head')[0].appendChild(el);
+			el.onload = () => {
+				frappe._assets_loaded.push(link);
+				resolve();
+			};
 		});
 	},
 	hide_message: function() {