Merge branch 'develop' into folder-navigation

.github/workflows/patch-mariadb-tests.yml

@@ -1,11 +1,6 @@
 name: Patch
 
-on:
-  pull_request:
-    paths-ignore:
-      - '**.js'
-      - '**.md'
-  workflow_dispatch:
+on: [pull_request, workflow_dispatch]
 
 jobs:
   test:

.github/workflows/server-mariadb-tests.yml

@@ -2,15 +2,9 @@ name: Server
 
 on:
   pull_request:
-    paths-ignore:
-      - '**.js'
-      - '**.md'
   workflow_dispatch:
   push:
     branches: [ develop ]
-    paths-ignore:
-      - '**.js'
-      - '**.md'
 
 jobs:
   test:

.github/workflows/server-postgres-tests.yml

@@ -2,9 +2,6 @@ name: Server
 
 on:
   pull_request:
-    paths-ignore:
-      - '**.js'
-      - '**.md'
   workflow_dispatch:
 
 jobs:

.github/workflows/ui-tests.yml

@@ -2,8 +2,6 @@ name: UI
 
 on:
   pull_request:
-    paths-ignore:
-      - '**.md'
   workflow_dispatch:
   push:
     branches: [ develop ]

.mergify.yml

@@ -1,9 +1,11 @@
 pull_request_rules:
   - name: Auto-close PRs on stable branch
     conditions:
-      - or:
-        - base=version-13
-        - base=version-12
+      - and:
+        - author!=surajshetty3416
+        - or:
+          - base=version-13
+          - base=version-12
     actions:
       close:
       comment:

cypress/integration/form_tour.js

@@ -20,10 +20,10 @@ context('Form Tour', () => {
 	it('navigates a form tour', () => {
 		open_test_form_tour();
 
-		cy.get('#driver-popover-item').should('be.visible');
+		cy.get('.frappe-driver').should('be.visible');
 		cy.get('.frappe-control[data-fieldname="first_name"]').as('first_name');
 		cy.get('@first_name').should('have.class', 'driver-highlighted-element');
-		cy.get('#driver-popover-item').findByRole('button', {name: 'Next'}).as('next_btn');
+		cy.get('.frappe-driver').findByRole('button', {name: 'Next'}).as('next_btn');
 
 		// next btn shouldn't move to next step, if first name is not entered
 		cy.get('@next_btn').click();
@@ -68,13 +68,13 @@ context('Form Tour', () => {
 		cy.get('.grid-row-open .frappe-control[data-fieldname="phone"]').as('phone');
 		cy.get('@phone').should('have.class', 'driver-highlighted-element');
 		// enter value in a table field
-		cy.fill_table_field('phone_nos', '1', 'phone', '1234567890');
+		let field = cy.fill_table_field('phone_nos', '1', 'phone', '1234567890');
+		field.blur();
 
 		// move to collapse row step
 		cy.wait(500);
-		cy.get('@next_btn').click();
+		cy.get('.driver-popover-title').contains('Test Title 4').siblings().get('@next_btn').click();
 		cy.wait(500);
-
 		// collapse row
 		cy.get('.grid-row-open .grid-collapse-row').click();
 		cy.wait(500);
@@ -82,7 +82,7 @@ context('Form Tour', () => {
 		// assert save btn is highlighted
 		cy.get('.primary-action').should('have.class', 'driver-highlighted-element');
 		cy.wait(500);
-		cy.get('#driver-popover-item').findByRole('button', {name: 'Save'}).should('be.visible');
+		cy.get('.frappe-driver').findByRole('button', {name: 'Save'}).should('be.visible');
 
 	});
 });

cypress/integration/timeline_email.js

@@ -8,14 +8,13 @@ context('Timeline Email', () => {
 	it('Adding new ToDo, adding email and verifying timeline content for email attachment, deleting attachment and ToDo', () => {
 		//Adding new ToDo
 		cy.click_listview_primary_button('Add ToDo');
-		cy.get('.custom-actions > .btn').trigger('click', {delay: 500});
-		cy.get('.row > .section-body > .form-column > form > .frappe-control > .form-group > .control-input-wrapper > .control-input > .ql-container > .ql-editor').eq(0).type('Test ToDo', {force: true});
+		cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500});
+		cy.fill_field("description", "Test ToDo",  "Text Editor");
 		cy.wait(500);
-		//cy.click_listview_primary_button('Save');
 		cy.get('.primary-action').contains('Save').click({force: true});
 		cy.wait(700);
 		cy.visit('/app/todo');
-		cy.get('.list-row > .level-left > .list-subject > .level-item.ellipsis > .ellipsis').eq(0).click();
+		cy.get('.list-row > .level-left > .list-subject').eq(0).click();
 
 		//Creating a new email
 		cy.get('.timeline-actions > .btn').click();
@@ -47,7 +46,7 @@ context('Timeline Email', () => {
 
 		//Removing the added attachment
 		cy.get('.attachment-row > .data-pill > .remove-btn > .icon').click();
-		cy.get('.modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary').contains('Yes').click();
+		cy.get('.modal-footer:visible > .standard-actions > .btn-primary').contains('Yes').click();
 
 		//To check if the removed attachment is shown in the timeline content
 		cy.get('.timeline-content').should('contain', 'Removed 72402.jpg');
@@ -55,17 +54,17 @@ context('Timeline Email', () => {
 
 		//To check if the discard button functionality in email is working correctly
 		cy.get('.timeline-actions > .btn').click();
-		cy.fill_field('recipients', 'test@example.com', 'MultiSelect'); 
+		cy.fill_field('recipients', 'test@example.com', 'MultiSelect');
 		cy.get('.modal-footer > .standard-actions > .btn-secondary').contains('Discard').click();
 		cy.wait(500);
 		cy.get('.timeline-actions > .btn').click();
 		cy.wait(500);
 		cy.get_field('recipients', 'MultiSelect').should('have.text', '');
-		cy.get('.modal.show > .modal-dialog > .modal-content > .modal-header > .modal-actions > .btn-modal-close > .icon').click();  
+		cy.get('.modal-header:visible > .modal-actions > .btn-modal-close > .icon').click();
 
 		//Deleting the added ToDo
-		cy.get('#page-ToDo > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .btn').click();
-		cy.get('.menu-btn-group > .dropdown-menu > li > .grey-link').eq(17).click();
-		cy.get('.modal.show > .modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary').click();
+		cy.get('.menu-btn-group:visible > .btn').click();
+		cy.get('.menu-btn-group:visible > .dropdown-menu > li > .dropdown-item').contains('Delete').click();
+		cy.get('.modal-footer:visible > .standard-actions > .btn-primary').click();
 	});
 });

cypress/integration/workspace.js

@@ -36,12 +36,12 @@ context('Workspace 2.0', () => {
 		cy.get('.codex-editor__redactor .ce-block');
 		cy.get('.custom-actions .inner-group-button[data-label="Add%20Block"]').click();
 		cy.get('.custom-actions .inner-group-button .dropdown-menu .block-menu-item-label').contains('Heading').click();
-		cy.get(".ce-block:last").find('h2').click({force: true}).type('Header');
+		cy.get(":focus").type('Header');
 		cy.get(".ce-block:last").find('.ce-header').should('exist');
 
 		cy.get('.custom-actions .inner-group-button[data-label="Add%20Block"]').click();
 		cy.get('.custom-actions .inner-group-button .dropdown-menu .block-menu-item-label').contains('Text').click();
-		cy.get(".ce-block:last").find('.ce-paragraph').click({force: true}).type('Paragraph text');
+		cy.get(":focus").type('Paragraph text');
 		cy.get(".ce-block:last").find('.ce-paragraph').should('exist');
 	});
 

frappe/automation/doctype/assignment_rule/test_assignment_rule.py

@@ -76,7 +76,7 @@ class TestAutoAssign(unittest.TestCase):
 		# clear 5 assignments for first user
 		# can't do a limit in "delete" since postgres does not support it
 		for d in frappe.get_all('ToDo', dict(reference_type = 'Note', owner = 'test@example.com'), limit=5):
-			frappe.db.sql("delete from tabToDo where name = %s", d.name)
+			frappe.db.delete("ToDo", {"name": d.name})
 
 		# add 5 more assignments
 		for i in range(5):
@@ -177,7 +177,7 @@ class TestAutoAssign(unittest.TestCase):
 		), 'owner'), 'test@example.com')
 
 	def check_assignment_rule_scheduling(self):
-		frappe.db.sql("DELETE FROM `tabAssignment Rule`")
+		frappe.db.delete("Assignment Rule")
 
 		days_1 = [dict(day = 'Sunday'), dict(day = 'Monday'), dict(day = 'Tuesday')]
 
@@ -204,7 +204,7 @@ class TestAutoAssign(unittest.TestCase):
 		), 'owner'), ['test3@example.com'])
 
 	def test_assignment_rule_condition(self):
-		frappe.db.sql("DELETE FROM `tabAssignment Rule`")
+		frappe.db.delete("Assignment Rule")
 
 		# Add expiry_date custom field
 		from frappe.custom.doctype.custom_field.custom_field import create_custom_field
@@ -253,7 +253,7 @@ class TestAutoAssign(unittest.TestCase):
 		assignment_rule.delete()
 
 def clear_assignments():
-	frappe.db.sql("delete from tabToDo where reference_type = 'Note'")
+	frappe.db.delete("ToDo", {"reference_type": "Note"})
 
 def get_assignment_rule(days, assign=None):
 	frappe.delete_doc_if_exists('Assignment Rule', 'For Note 1')

frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py

@@ -7,7 +7,7 @@ import unittest
 
 class TestMilestoneTracker(unittest.TestCase):
 	def test_milestone(self):
-		frappe.db.sql('delete from `tabMilestone Tracker`')
+		frappe.db.delete("Milestone Tracker")
 
 		frappe.cache().delete_key('milestone_tracker_map')
 
@@ -44,5 +44,5 @@ class TestMilestoneTracker(unittest.TestCase):
 		self.assertEqual(milestones[0].value, 'Closed')
 
 		# cleanup
-		frappe.db.sql('delete from tabMilestone')
+		frappe.db.delete("Milestone")
 		milestone_tracker.delete()

frappe/commands/utils.py

@@ -486,15 +486,26 @@ frappe.db.connect()
 
 
 @click.command('console')
+@click.option(
+	'--autoreload',
+	is_flag=True,
+	help="Reload changes to code automatically"
+)
 @pass_context
-def console(context):
+def console(context, autoreload=False):
 	"Start ipython console for a site"
 	site = get_site(context)
 	frappe.init(site=site)
 	frappe.connect()
 	frappe.local.lang = frappe.db.get_default("lang")
 
-	import IPython
+	from IPython.terminal.embed import InteractiveShellEmbed
+
+	terminal = InteractiveShellEmbed()
+	if autoreload:
+		terminal.extension_manager.load_extension("autoreload")
+		terminal.run_line_magic("autoreload", "2")
+
 	all_apps = frappe.get_installed_apps()
 	failed_to_import = []
 
@@ -509,7 +520,9 @@ def console(context):
 	if failed_to_import:
 		print("\nFailed to import:\n{}".format(", ".join(failed_to_import)))
 
-	IPython.embed(display_banner="", header="", colors="neutral")
+	terminal.colors = "neutral"
+	terminal.display_banner = False
+	terminal()
 
 
 @click.command('run-tests')

frappe/core/doctype/access_log/access_log.py

@@ -29,4 +29,5 @@ def make_access_log(doctype=None, document=None, method=None, file_type=None,
 	doc.insert(ignore_permissions=True)
 
 	# `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview`
-	frappe.db.commit()
+	if frappe.request and frappe.request.method == 'GET':
+		frappe.db.commit()

frappe/core/doctype/comment/test_comment.py

@@ -30,7 +30,7 @@ class TestComment(unittest.TestCase):
 		from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
 		test_blog = make_test_blog()
 
-		frappe.db.sql("delete from `tabComment` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Comment", {"reference_doctype": "Blog Post"})
 
 		from frappe.templates.includes.comments.comments import add_comment
 		add_comment('Good comment with 10 chars', 'test@test.com', 'Good Tester',
@@ -41,7 +41,7 @@ class TestComment(unittest.TestCase):
 			reference_name = test_blog.name
 		))[0].published, 1)
 
-		frappe.db.sql("delete from `tabComment` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Comment", {"reference_doctype": "Blog Post"})
 
 		add_comment('pleez vizits my site http://mysite.com', 'test@test.com', 'bad commentor',
 			'Blog Post', test_blog.name, test_blog.route)

frappe/core/doctype/feedback/test_feedback.py

@@ -9,7 +9,7 @@ class TestFeedback(unittest.TestCase):
 		from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
 		test_blog = make_test_blog()
 
-		frappe.db.sql("delete from `tabFeedback` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"})
 
 		from frappe.templates.includes.feedback.feedback import add_feedback, update_feedback
 		feedback = add_feedback('Blog Post', test_blog.name, 5, 'New feedback')
@@ -22,6 +22,6 @@ class TestFeedback(unittest.TestCase):
 		self.assertEqual(updated_feedback.feedback, 'Updated feedback')
 		self.assertEqual(updated_feedback.rating, 6)
 
-		frappe.db.sql("delete from `tabFeedback` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"})
 
 		test_blog.delete()

frappe/core/doctype/file/file.py

@@ -21,11 +21,11 @@ import zipfile
 import requests
 import requests.exceptions
 from PIL import Image, ImageFile, ImageOps
-from io import StringIO
+from io import BytesIO
 from urllib.parse import quote, unquote
 
 import frappe
-from frappe import _, conf
+from frappe import _, conf, safe_decode
 from frappe.model.document import Document
 from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, get_hook_method, random_string, strip
 from frappe.utils.image import strip_exif_data, optimize_image
@@ -257,8 +257,7 @@ class File(Document):
 			with open(get_files_path(file_name, is_private=self.is_private), "rb") as f:
 				self.content_hash = get_content_hash(f.read())
 		except IOError:
-			frappe.msgprint(_("File {0} does not exist").format(self.file_url))
-			raise
+			frappe.throw(_("File {0} does not exist").format(self.file_url))
 
 	def on_trash(self):
 		if self.is_home_folder or self.is_attachments_folder:
@@ -270,16 +269,12 @@ class File(Document):
 
 	def make_thumbnail(self, set_as_thumbnail=True, width=300, height=300, suffix="small", crop=False):
 		if self.file_url:
-			if self.file_url.startswith("/files"):
-				try:
+			try:
+				if self.file_url.startswith(("/files", "/private/files")):
 					image, filename, extn = get_local_image(self.file_url)
-				except IOError:
-					return
-
-			else:
-				try:
+				else:
 					image, filename, extn = get_web_image(self.file_url)
-				except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError):
+			except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError):
 					return
 
 			size = width, height
@@ -289,16 +284,13 @@ class File(Document):
 				image.thumbnail(size, Image.ANTIALIAS)
 
 			thumbnail_url = filename + "_" + suffix + "." + extn
-
 			path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/")))
 
 			try:
 				image.save(path)
-
 				if set_as_thumbnail:
 					self.db_set("thumbnail_url", thumbnail_url)
 
-				self.db_set("thumbnail_url", thumbnail_url)
 			except IOError:
 				frappe.msgprint(_("Unable to write file format for {0}").format(path))
 				return
@@ -326,12 +318,10 @@ class File(Document):
 
 	def unzip(self):
 		'''Unzip current file and replace it by its children'''
-		if not ".zip" in self.file_name:
-			frappe.msgprint(_("Not a zip file"))
-			return
+		if not self.file_url.endswith(".zip"):
+			frappe.throw(_("{0} is not a zip file").format(self.file_name))
 
-		zip_path = frappe.get_site_path(self.file_url.strip('/'))
-		base_url = os.path.dirname(self.file_url)
+		zip_path = self.get_full_path()
 
 		files = []
 		with zipfile.ZipFile(zip_path) as z:
@@ -359,10 +349,6 @@ class File(Document):
 		return files
 
 
-	def get_file_url(self):
-		data = frappe.db.get_value("File", self.file_data_name, ["file_name", "file_url"], as_dict=True)
-		return data.file_url or data.file_name
-
 	def exists_on_disk(self):
 		exists = os.path.exists(self.get_full_path())
 		return exists
@@ -431,47 +417,6 @@ class File(Document):
 
 		return get_files_path(self.file_name, is_private=self.is_private)
 
-	def get_file_doc(self):
-		'''returns File object (Document) from given parameters or form_dict'''
-		r = frappe.form_dict
-
-		if self.file_url is None: self.file_url = r.file_url
-		if self.file_name is None: self.file_name = r.file_name
-		if self.attached_to_doctype is None: self.attached_to_doctype = r.doctype
-		if self.attached_to_name is None: self.attached_to_name = r.docname
-		if self.attached_to_field is None: self.attached_to_field = r.docfield
-		if self.folder is None: self.folder = r.folder
-		if self.is_private is None: self.is_private = r.is_private
-
-		if r.filedata:
-			file_doc = self.save_uploaded()
-
-		elif r.file_url:
-			file_doc = self.save()
-
-		return file_doc
-
-
-	def save_uploaded(self):
-		self.content = self.get_uploaded_content()
-		if self.content:
-			return self.save()
-		else:
-			raise Exception
-
-	def get_uploaded_content(self):
-		# should not be unicode when reading a file, hence using frappe.form
-		if 'filedata' in frappe.form_dict:
-			if "," in frappe.form_dict.filedata:
-				frappe.form_dict.filedata = frappe.form_dict.filedata.rsplit(",", 1)[1]
-			frappe.uploaded_content = base64.b64decode(frappe.form_dict.filedata)
-			return frappe.uploaded_content
-		elif self.content:
-			return self.content
-		frappe.msgprint(_('No file attached'))
-		return None
-
-
 	def save_file(self, content=None, decode=False, ignore_existing_file_check=False):
 		file_exists = False
 		self.content = content
@@ -539,14 +484,6 @@ class File(Document):
 			'file_url': self.file_url
 		}
 
-	def get_file_data_from_hash(self):
-		for name in frappe.db.sql_list("select name from `tabFile` where content_hash=%s and is_private=%s",
-			(self.content_hash, self.is_private)):
-			b = frappe.get_doc('File', name)
-			return {k: b.get(k) for k in frappe.get_hooks()['write_file_keys']}
-		return False
-
-
 	def check_max_file_size(self):
 		max_file_size = get_max_file_size()
 		file_size = len(self.content)
@@ -621,7 +558,8 @@ def create_new_folder(file_name, folder):
 	file.file_name = file_name
 	file.is_folder = 1
 	file.folder = folder
-	file.insert()
+	file.insert(ignore_if_duplicate=True)
+	return file
 
 @frappe.whitelist()
 def move_file(file_list, new_parent, old_parent):
@@ -672,7 +610,7 @@ def get_local_image(file_url):
 	try:
 		image = Image.open(file_path)
 	except IOError:
-		frappe.msgprint(_("Unable to read file format for {0}").format(file_url), raise_exception=True)
+		frappe.throw(_("Unable to read file format for {0}").format(file_url))
 
 	content = None
 
@@ -704,7 +642,7 @@ def get_web_image(file_url):
 		raise
 
 	try:
-		image = Image.open(StringIO(frappe.safe_decode(r.content)))
+		image = Image.open(BytesIO(r.content))
 	except Exception as e:
 		frappe.msgprint(_("Image link '{0}' is not valid").format(file_url), raise_exception=e)
 
@@ -740,48 +678,12 @@ def delete_file(path):
 			os.remove(path)
 
 
-def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_delete=False, delete_permanently=False):
-	"""Remove file and File entry"""
-	file_name = None
-	if not (attached_to_doctype and attached_to_name):
-		attached = frappe.db.get_value("File", fid,
-			["attached_to_doctype", "attached_to_name", "file_name"])
-		if attached:
-			attached_to_doctype, attached_to_name, file_name = attached
-
-	ignore_permissions, comment = False, None
-	if attached_to_doctype and attached_to_name and not from_delete:
-		doc = frappe.get_doc(attached_to_doctype, attached_to_name)
-		ignore_permissions = doc.has_permission("write") or False
-		if frappe.flags.in_web_form:
-			ignore_permissions = True
-		if not file_name:
-			file_name = frappe.db.get_value("File", fid, "file_name")
-		comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name))
-		frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions, delete_permanently=delete_permanently)
-
-	return comment
 
 
 def get_max_file_size():
 	return cint(conf.get('max_file_size')) or 10485760
 
 
-def remove_all(dt, dn, from_delete=False, delete_permanently=False):
-	"""remove all files in a transaction"""
-	try:
-		for fid in frappe.db.sql_list("""select name from `tabFile` where
-			attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)):
-			if from_delete:
-				# If deleting a doc, directly delete files
-				frappe.delete_doc("File", fid, ignore_permissions=True, delete_permanently=delete_permanently)
-			else:
-				# Removes file and adds a comment in the document it is attached to
-				remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn,
-					from_delete=from_delete, delete_permanently=delete_permanently)
-	except Exception as e:
-		if e.args[0]!=1054: raise # (temp till for patched)
-
 
 def has_permission(doc, ptype=None, user=None):
 	has_access = False
@@ -827,6 +729,7 @@ def remove_file_by_url(file_url, doctype=None, name=None):
 		fid = frappe.db.get_value("File", {"file_url": file_url})
 
 	if fid:
+		from frappe.utils.file_manager import remove_file
 		return remove_file(fid=fid)
 
 
@@ -886,15 +789,13 @@ def extract_images_from_html(doc, content):
 		if b"," in content:
 			content = content.split(b",")[1]
 		content = base64.b64decode(content)
-		
+
 		content = optimize_image(content, mtype)
 
 		if "filename=" in headers:
 			filename = headers.split("filename=")[-1]
+			filename = safe_decode(filename).split(";")[0]
 
-			# decode filename
-			if not isinstance(filename, str):
-				filename = str(filename, 'utf-8')
 		else:
 			filename = get_random_filename(content_type=mtype)
 
@@ -922,12 +823,9 @@ def extract_images_from_html(doc, content):
 	return content
 
 
-def get_random_filename(extn=None, content_type=None):
-	if extn:
-		if not extn.startswith("."):
-			extn = "." + extn
-
-	elif content_type:
+def get_random_filename(content_type=None):
+	extn = None
+	if content_type:
 		extn = mimetypes.guess_extension(content_type)
 
 	return random_string(7) + (extn or "")
@@ -938,7 +836,7 @@ def unzip_file(name):
 	'''Unzip the given file and make file records for each of the extracted files'''
 	file_obj = frappe.get_doc('File', name)
 	files = file_obj.unzip()
-	return len(files)
+	return files
 
 @frappe.whitelist()
 def optimize_saved_image(doc_name):
@@ -979,13 +877,6 @@ def get_attached_images(doctype, names):
 	return out
 
 
-@frappe.whitelist()
-def validate_filename(filename):
-	from frappe.utils import now_datetime
-	timestamp = now_datetime().strftime(" %Y-%m-%d %H:%M:%S")
-	fname = get_file_name(filename, timestamp)
-	return fname
-
 @frappe.whitelist()
 def get_files_in_folder(folder, start=0, page_length=20):
 	start = cint(start)

frappe/core/doctype/file/test_file.py

@@ -2,11 +2,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # See license.txt
 import base64
+import json
 import frappe
 import os
 import unittest
 from frappe import _
-from frappe.core.doctype.file.file import move_file, get_files_in_folder
+from frappe.core.doctype.file.file import get_attached_images, move_file, get_files_in_folder, unzip_file
 from frappe.utils import get_files_path
 # test_records = frappe.get_test_records('File')
 
@@ -365,6 +366,80 @@ class TestFile(unittest.TestCase):
 		file1.file_url = '/private/files/parent_dir2.txt'
 		file1.save()
 
+	def test_file_url_validation(self):
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'logo',
+			"file_url": 'https://frappe.io/files/frappe.png'
+		})
+
+		self.assertIsNone(test_file.validate())
+
+		# bad path
+		test_file.file_url = "/usr/bin/man"
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "URL must start with http:// or https://", test_file.validate)
+
+		test_file.file_url = None
+		test_file.file_name = "/usr/bin/man"
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "There is some problem with the file url", test_file.validate)
+
+		test_file.file_url = None
+		test_file.file_name = "_file"
+		self.assertRaisesRegex(IOError, "does not exist", test_file.validate)
+
+		test_file.file_url = None
+		test_file.file_name = "/private/files/_file"
+		self.assertRaisesRegex(IOError, "does not exist", test_file.validate)
+
+	def test_make_thumbnail(self):
+		# test web image
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'logo',
+			"file_url": frappe.utils.get_url('/_test/assets/image.jpg'),
+		}).insert(ignore_permissions=True)
+
+		test_file.make_thumbnail()
+		self.assertEquals(test_file.thumbnail_url, '/files/image_small.jpg')
+
+		# test local image
+		test_file.db_set('thumbnail_url', None)
+		test_file.reload()
+		test_file.file_url = "/files/image_small.jpg"
+		test_file.make_thumbnail(suffix="xs", crop=True)
+		self.assertEquals(test_file.thumbnail_url, '/files/image_small_xs.jpg')
+
+		frappe.clear_messages()
+		test_file.db_set('thumbnail_url', None)
+		test_file.reload()
+		test_file.file_url = frappe.utils.get_url('unknown.jpg')
+		test_file.make_thumbnail(suffix="xs")
+		self.assertEqual(json.loads(frappe.message_log[0]), {"message": f"File '{frappe.utils.get_url('unknown.jpg')}' not found"})
+		self.assertEquals(test_file.thumbnail_url, None)
+
+	def test_file_unzip(self):
+		file_path = frappe.get_app_path('frappe', 'www/_test/assets/file.zip')
+		public_file_path = frappe.get_site_path('public', 'files')
+		try:
+			import shutil
+			shutil.copy(file_path, public_file_path)
+		except Exception:
+			pass
+
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_url": '/files/file.zip',
+		}).insert(ignore_permissions=True)
+
+		self.assertListEqual([file.file_name for file in unzip_file(test_file.name)],
+			['css_asset.css', 'image.jpg', 'js_asset.min.js'])
+
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_url": frappe.utils.get_url('/_test/assets/image.jpg'),
+		}).insert(ignore_permissions=True)
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, 'not a zip file', test_file.unzip)
+
 class TestAttachment(unittest.TestCase):
 	test_doctype = 'Test For Attachment'
 
@@ -469,3 +544,28 @@ class TestAttachmentsAccess(unittest.TestCase):
 
 		frappe.set_user('Administrator')
 		frappe.db.rollback()
+
+
+class TestFileUtils(unittest.TestCase):
+	def test_extract_images_from_doc(self):
+		# with filename in data URI
+		todo = frappe.get_doc({
+			"doctype": "ToDo",
+			"description": 'Test <img src="data:image/png;filename=pix.png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">'
+		}).insert()
+		self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name}))
+		self.assertIn('<img src="/files/pix.png">', todo.description)
+		self.assertListEqual(get_attached_images('ToDo', [todo.name])[todo.name], ['/files/pix.png'])
+
+		# without filename in data URI
+		todo = frappe.get_doc({
+			"doctype": "ToDo",
+			"description": 'Test <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">'
+		}).insert()
+		filename = frappe.db.exists("File", {"attached_to_name": todo.name})
+		self.assertIn(f'<img src="{frappe.get_doc("File", filename).file_url}', todo.description)
+
+	def test_create_new_folder(self):
+		from frappe.core.doctype.file.file import create_new_folder
+		folder = create_new_folder('test_folder', 'Home')
+		self.assertTrue(folder.is_folder)

frappe/core/doctype/prepared_report/prepared_report.py

@@ -11,8 +11,6 @@ from frappe.desk.query_report import generate_report_result
 from frappe.model.document import Document
 from frappe.utils import gzip_compress, gzip_decompress
 from frappe.utils.background_jobs import enqueue
-from frappe.core.doctype.file.file import remove_all
-
 
 class PreparedReport(Document):
 	def before_insert(self):

frappe/core/doctype/role_profile/role_profile.py

@@ -3,6 +3,7 @@
 # For license information, please see license.txt
 
 from frappe.model.document import Document
+import frappe
 
 class RoleProfile(Document):
 	def autoname(self):
@@ -11,5 +12,9 @@ class RoleProfile(Document):
 
 	def on_update(self):
 		""" Changes in role_profile reflected across all its user """
-		from frappe.core.doctype.user.user import update_roles
-		update_roles(self.name)
+		users = frappe.get_all('User', filters={'role_profile_name': self.name})
+		roles = [role.role for role in self.roles]
+		for d in users:
+			user = frappe.get_doc('User', d)
+			user.set('roles', [])
+			user.add_roles(*roles)

frappe/core/doctype/role_profile/test_role_profile.py

@@ -8,6 +8,7 @@ test_dependencies = ['Role']
 
 class TestRoleProfile(unittest.TestCase):
 	def test_make_new_role_profile(self):
+		frappe.delete_doc_if_exists('Role Profile', 'Test 1', force=1)
 		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
 
 		self.assertEqual(new_role_profile.role_profile, 'Test 1')
@@ -19,7 +20,25 @@ class TestRoleProfile(unittest.TestCase):
 		new_role_profile.save()
 		self.assertEqual(new_role_profile.roles[0].role, '_Test Role 2')
 
+		# user with a role profile
+		random_user = frappe.mock("email")
+		random_user_name = frappe.mock("name")
+
+		random_user = frappe.get_doc({
+			"doctype": "User",
+			"email": random_user,
+			"enabled": 1,
+			"first_name": random_user_name,
+			"new_password": "Eastern_43A1W",
+			"role_profile_name": 'Test 1'
+		}).insert(ignore_permissions=True, ignore_if_duplicate=True)
+		self.assertListEqual([role.role for role in random_user.roles], [role.role for role in new_role_profile.roles])
+
 		# clear roles
 		new_role_profile.roles = []
 		new_role_profile.save()
 		self.assertEqual(new_role_profile.roles, [])
+
+		# user roles with the role profile should also be updated
+		random_user.reload()
+		self.assertListEqual(random_user.roles, [])

frappe/core/doctype/translation/test_translation.py

@@ -8,7 +8,7 @@ from frappe import _
 
 class TestTranslation(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql('delete from tabTranslation')
+		frappe.db.delete("Translation")
 
 	def tearDown(self):
 		frappe.local.lang = 'en'

frappe/core/doctype/user/test_records.json

@@ -70,5 +70,19 @@
     "role": "System Manager"
    }
   ]
- }
+ },
+ {
+    "doctype": "User",
+    "email": "testpassword@example.com",
+    "enabled": 1,
+    "first_name": "_Test",
+    "new_password": "Eastern_43A1W",
+    "roles": [
+     {
+      "doctype": "Has Role",
+      "parentfield": "roles",
+      "role": "System Manager"
+     }
+    ]
+   }
 ]

frappe/core/doctype/user/test_user.py

@@ -1,16 +1,18 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-import frappe, unittest, uuid
+import json
+import unittest
+from unittest.mock import patch
 
-from frappe.model.delete_doc import delete_doc
-from frappe.utils.data import today, add_to_date
-from frappe import _dict
-from frappe.utils import get_url
-from frappe.core.doctype.user.user import get_total_users
-from frappe.core.doctype.user.user import MaxUsersReachedError, test_password_strength
-from frappe.core.doctype.user.user import extract_mentions
+import frappe
+import frappe.exceptions
+from frappe.core.doctype.user.user import (extract_mentions, reset_password,
+	sign_up, test_password_strength, update_password, verify_password)
 from frappe.frappeclient import FrappeClient
+from frappe.model.delete_doc import delete_doc
+from frappe.utils import get_url
 
+user_module = frappe.core.doctype.user.user
 test_records = frappe.get_test_records('User')
 
 class TestUser(unittest.TestCase):
@@ -23,7 +25,7 @@ class TestUser(unittest.TestCase):
 
 	def test_user_type(self):
 		new_user = frappe.get_doc(dict(doctype='User', email='test-for-type@example.com',
-			first_name='Tester')).insert()
+			first_name='Tester')).insert(ignore_if_duplicate=True)
 		self.assertEqual(new_user.user_type, 'Website User')
 
 		# social login userid for frappe
@@ -52,7 +54,7 @@ class TestUser(unittest.TestCase):
 	def test_delete(self):
 		frappe.get_doc("User", "test@example.com").add_roles("_Test Role 2")
 		self.assertRaises(frappe.LinkExistsError, delete_doc, "Role", "_Test Role 2")
-		frappe.db.sql("""delete from `tabHas Role` where role='_Test Role 2'""")
+		frappe.db.delete("Has Role", {"role": "_Test Role 2"})
 		delete_doc("Role","_Test Role 2")
 
 		if frappe.db.exists("User", "_test@example.com"):
@@ -119,40 +121,9 @@ class TestUser(unittest.TestCase):
 		# system manager now added by Administrator
 		self.assertTrue("System Manager" in [d.role for d in me.get("roles")])
 
-	# def test_deny_multiple_sessions(self):
-	#	from frappe.installer import update_site_config
-	# 	clear_limit('users')
-	#
-	# 	# allow one session
-	# 	user = frappe.get_doc('User', 'test@example.com')
-	# 	user.simultaneous_sessions = 1
-	# 	user.new_password = 'Eastern_43A1W'
-	# 	user.save()
-	#
-	# 	def test_request(conn):
-	# 		value = conn.get_value('User', 'first_name', {'name': 'test@example.com'})
-	# 		self.assertTrue('first_name' in value)
-	#
-	# 	from frappe.frappeclient import FrappeClient
-	# 	update_site_config('deny_multiple_sessions', 0)
-	#
-	# 	conn1 = FrappeClient(get_url(), "test@example.com", "Eastern_43A1W", verify=False)
-	# 	test_request(conn1)
-	#
-	# 	conn2 = FrappeClient(get_url(), "test@example.com", "Eastern_43A1W", verify=False)
-	# 	test_request(conn2)
-	#
-	# 	update_site_config('deny_multiple_sessions', 1)
-	# 	conn3 = FrappeClient(get_url(), "test@example.com", "Eastern_43A1W", verify=False)
-	# 	test_request(conn3)
-	#
-	# 	# first connection should fail
-	# 	test_request(conn1)
-
-
 	def test_delete_user(self):
 		new_user = frappe.get_doc(dict(doctype='User', email='test-for-delete@example.com',
-			first_name='Tester Delete User')).insert()
+			first_name='Tester Delete User')).insert(ignore_if_duplicate=True)
 		self.assertEqual(new_user.user_type, 'Website User')
 
 		# role with desk access
@@ -174,7 +145,7 @@ class TestUser(unittest.TestCase):
 		self.assertFalse(frappe.db.exists('User', new_user.name))
 
 	def test_password_strength(self):
-		# Test Password without Password Strenth Policy
+		# Test Password without Password Strength Policy
 		frappe.db.set_value("System Settings", "System Settings", "enable_password_policy", 0)
 
 		# password policy is disabled, test_password_strength should be ignored
@@ -193,6 +164,17 @@ class TestUser(unittest.TestCase):
 		result = test_password_strength("Eastern_43A1W")
 		self.assertEqual(result['feedback']['password_policy_validation_passed'], True)
 
+
+		# test password strength while saving user with new password
+		user = frappe.get_doc("User", "test@example.com")
+		frappe.flags.in_test = False
+		user.new_password = "password"
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "Invalid Password", user.save)
+		user.reload()
+		user.new_password = "Eastern_43A1W"
+		user.save()
+		frappe.flags.in_test = True
+
 	def test_comment_mentions(self):
 		comment = '''
 			<span class="mention" data-id="test.comment@example.com" data-value="Test" data-denotation-char="@">
@@ -227,6 +209,7 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(extract_mentions(comment)[0], "test_user@example.com")
 		self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com")
 
+		frappe.delete_doc("User Group", "Team")
 		doc = frappe.get_doc({
 			'doctype': 'User Group',
 			'name': 'Team',
@@ -236,14 +219,18 @@ class TestUser(unittest.TestCase):
 				'user': 'test1@example.com'
 			}]
 		})
-		doc.insert(ignore_if_duplicate=True)
+
+		doc.insert()
 
 		comment = '''
 			<div>
 				Testing comment for
 				<span class="mention" data-id="Team" data-value="Team" data-is-group="true" data-denotation-char="@">
 					<span><span class="ql-mention-denotation-char">@</span>Team</span>
-				</span>
+				</span> and
+				<span class="mention" data-id="Unknown Team" data-value="Unknown Team" data-is-group="true" data-denotation-char="@">
+					<span><span class="ql-mention-denotation-char">@</span>Unknown Team</span>
+				</span><!-- this should be ignored-->
 				please check
 			</div>
 		'''
@@ -267,32 +254,125 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(res1.status_code, 200)
 		self.assertEqual(res2.status_code, 417)
 
-	# def test_user_rollback(self):
-	# 	"""
-	#	FIXME: This is failing with PR #12693 as Rollback can't happen if notifications sent on user creation.
-	#	Make sure that notifications disabled.
-	# 	"""
-	# 	frappe.db.commit()
-	# 	frappe.db.begin()
-	# 	user_id = str(uuid.uuid4())
-	# 	email = f'{user_id}@example.com'
-	# 	try:
-	# 		frappe.flags.in_import = True  # disable throttling
-	# 		frappe.get_doc(dict(
-	# 			doctype='User',
-	# 			email=email,
-	# 			first_name=user_id,
-	# 		)).insert()
-	# 	finally:
-	# 		frappe.flags.in_import = False
+	def test_user_rename(self):
+		old_name = "test_user_rename@example.com"
+		new_name = "test_user_rename_new@example.com"
+		user = frappe.get_doc({
+			"doctype": "User",
+			"email": old_name,
+			"enabled": 1,
+			"first_name": "_Test",
+			"new_password": "Eastern_43A1W",
+			"roles": [
+				{
+					"doctype": "Has Role",
+					"parentfield": "roles",
+					"role": "System Manager"
+				}]
+		}).insert(ignore_permissions=True, ignore_if_duplicate=True)
 
-	# 	# Check user has been added
-	# 	self.assertIsNotNone(frappe.db.get("User", {"email": email}))
+		frappe.rename_doc('User', user.name, new_name)
+		self.assertTrue(frappe.db.exists("Notification Settings", new_name))
+
+		frappe.delete_doc("User", new_name)
+
+	def test_signup(self):
+		import frappe.website.utils
+		random_user = frappe.mock('email')
+		random_user_name = frappe.mock('name')
+		# disabled signup
+		with patch.object(user_module, "is_signup_disabled", return_value=True):
+			self.assertRaisesRegex(frappe.exceptions.ValidationError, "Sign Up is disabled",
+				sign_up, random_user, random_user_name, "/signup")
+
+		self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (1, "Please check your email for verification"))
+		self.assertEqual(frappe.cache().hget('redirect_after_login', random_user), "/welcome")
+
+		# re-register
+		self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (0, "Already Registered"))
+
+		# disabled user
+		user = frappe.get_doc("User", random_user)
+		user.enabled = 0
+		user.save()
+
+		self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (0, "Registered but disabled"))
+
+		# throttle user creation
+		with patch.object(user_module.frappe.db, "get_creation_count", return_value=301):
+			self.assertRaisesRegex(frappe.exceptions.ValidationError, "Throttled",
+				sign_up, frappe.mock('email'), random_user_name, "/signup")
+
+
+	def test_reset_password(self):
+		from frappe.auth import CookieManager, LoginManager
+		from frappe.utils import set_request
+		old_password = "Eastern_43A1W"
+		new_password = "easy_password"
+
+		set_request(path="/random")
+		frappe.local.cookie_manager = CookieManager()
+		frappe.local.login_manager = LoginManager()
+
+		frappe.set_user("testpassword@example.com")
+		test_user = frappe.get_doc("User", "testpassword@example.com")
+		test_user.reset_password()
+		self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/app")
+		self.assertEqual(update_password(new_password, key="wrong_key"), "The Link specified has either been used before or Invalid")
+
+		# password verification should fail with old password
+		self.assertRaises(frappe.exceptions.AuthenticationError, verify_password, old_password)
+		verify_password(new_password)
+
+		# reset password
+		update_password(old_password, old_password=new_password)
+
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "Invalid key type", update_password, "test", 1, ['like', '%'])
+
+		password_strength_response = {
+			"feedback": {
+				"password_policy_validation_passed": False,
+				"suggestions": ["Fix password"]
+			}
+		}
+
+		# password strength failure test
+		with patch.object(user_module, "test_password_strength", return_value=password_strength_response):
+			self.assertRaisesRegex(frappe.exceptions.ValidationError, "Fix password", update_password, new_password, 0, test_user.reset_password_key)
+
+
+		# test redirect URL for website users
+		frappe.set_user("test2@example.com")
+		self.assertEqual(update_password(new_password, old_password=old_password), "/")
+		# reset password
+		update_password(old_password, old_password=new_password)
+
+		# test API endpoint
+		with patch.object(user_module.frappe, 'sendmail') as sendmail:
+			frappe.clear_messages()
+			test_user = frappe.get_doc("User", "test2@example.com")
+			self.assertEqual(reset_password(user="test2@example.com"), None)
+			test_user.reload()
+			self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/")
+			update_password(old_password, old_password=new_password)
+			self.assertEqual(json.loads(frappe.message_log[0]), {"message": "Password reset instructions have been sent to your email"})
+		sendmail.assert_called_once()
+		self.assertEqual(sendmail.call_args[1]["recipients"], "test2@example.com")
+
+		self.assertEqual(reset_password(user="test2@example.com"), None)
+		self.assertEqual(reset_password(user="Administrator"), "not allowed")
+		self.assertEqual(reset_password(user="random"), "not found")
+
+	def test_user_onload_modules(self):
+		from frappe.config import get_modules_from_all_apps
+		from frappe.desk.form.load import getdoc
+		frappe.response.docs = []
+		getdoc("User", "Administrator")
+		doc = frappe.response.docs[0]
+		self.assertListEqual(doc.get("__onload").get('all_modules', []),
+			[m.get("module_name") for m in get_modules_from_all_apps()])
 
-	# 	# Check that rollback works
-	# 	frappe.db.rollback()
-	# 	self.assertIsNone(frappe.db.get("User", {"email": email}))
 
 def delete_contact(user):
-	frappe.db.sql("DELETE FROM `tabContact` WHERE `email_id`= %s", user)
-	frappe.db.sql("DELETE FROM `tabContact Email` WHERE `email_id`= %s", user)
+	frappe.db.delete("Contact", {"email_id": user})
+	frappe.db.delete("Contact Email", {"email_id": user})

frappe/core/doctype/user/user.js

@@ -166,7 +166,7 @@ frappe.ui.form.on('User', {
 
 			frm.add_custom_button(__("Reset OTP Secret"), function() {
 				frappe.call({
-					method: "frappe.core.doctype.user.user.reset_otp_secret",
+					method: "frappe.twofactor.reset_otp_secret",
 					args: {
 						"user": frm.doc.name
 					}

frappe/core/doctype/user/user.py

@@ -15,17 +15,11 @@ from frappe.desk.doctype.notification_settings.notification_settings import crea
 from frappe.utils.user import get_system_managers
 from frappe.website.utils import is_signup_disabled
 from frappe.rate_limiter import rate_limit
-from frappe.utils.background_jobs import enqueue
 from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype
 
 
 STANDARD_USERS = ("Guest", "Administrator")
 
-
-class MaxUsersReachedError(frappe.ValidationError):
-	pass
-
-
 class User(Document):
 	__new_password = None
 
@@ -56,8 +50,6 @@ class User(Document):
 		frappe.cache().delete_key('enabled_users')
 
 	def validate(self):
-		self.check_demo()
-
 		# clear new password
 		self.__new_password = self.new_password
 		self.new_password = ""
@@ -137,10 +129,6 @@ class User(Document):
 		"""Returns true if current user is the session user"""
 		return self.name == frappe.session.user
 
-	def check_demo(self):
-		if frappe.session.user == 'demo@erpnext.com':
-			frappe.throw(_('Cannot change user details in demo. Please signup for a new account at https://erpnext.com'), title=_('Not Allowed'))
-
 	def set_full_name(self):
 		self.full_name = " ".join(filter(None, [self.first_name, self.last_name]))
 
@@ -398,7 +386,6 @@ class User(Document):
 
 
 	def before_rename(self, old_name, new_name, merge=False):
-		self.check_demo()
 		frappe.clear_cache(user=old_name)
 		self.validate_rename(old_name, new_name)
 
@@ -718,85 +705,6 @@ def get_email_awaiting(user):
 				where parent = %(user)s""",{"user":user})
 		return False
 
-@frappe.whitelist(allow_guest=False)
-def set_email_password(email_account, user, password):
-	account = frappe.get_doc("Email Account", email_account)
-	if account.awaiting_password:
-		account.awaiting_password = 0
-		account.password = password
-		try:
-			account.save(ignore_permissions=True)
-		except Exception:
-			frappe.db.rollback()
-			return False
-
-	return True
-
-def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing):
-	""" setup email inbox for user """
-	def add_user_email(user):
-		user = frappe.get_doc("User", user)
-		row = user.append("user_emails", {})
-
-		row.email_id = email_id
-		row.email_account = email_account
-		row.awaiting_password = awaiting_password or 0
-		row.enable_outgoing = enable_outgoing or 0
-
-		user.save(ignore_permissions=True)
-
-	udpate_user_email_settings = False
-	if not all([email_account, email_id]):
-		return
-
-	user_names = frappe.db.get_values("User", { "email": email_id }, as_dict=True)
-	if not user_names:
-		return
-
-	for user in user_names:
-		user_name = user.get("name")
-
-		# check if inbox is alreay configured
-		user_inbox = frappe.db.get_value("User Email", {
-			"email_account": email_account,
-			"parent": user_name
-		}, ["name"]) or None
-
-		if not user_inbox:
-			add_user_email(user_name)
-		else:
-			# update awaiting password for email account
-			udpate_user_email_settings = True
-
-	if udpate_user_email_settings:
-		frappe.db.sql("""UPDATE `tabUser Email` SET awaiting_password = %(awaiting_password)s,
-			enable_outgoing = %(enable_outgoing)s WHERE email_account = %(email_account)s""", {
-				"email_account": email_account,
-				"enable_outgoing": enable_outgoing,
-				"awaiting_password": awaiting_password or 0
-			})
-	else:
-		users = " and ".join([frappe.bold(user.get("name")) for user in user_names])
-		frappe.msgprint(_("Enabled email inbox for user {0}").format(users))
-
-	ask_pass_update()
-
-def remove_user_email_inbox(email_account):
-	""" remove user email inbox settings if email account is deleted """
-	if not email_account:
-		return
-
-	users = frappe.get_all("User Email", filters={
-		"email_account": email_account
-	}, fields=["parent as name"])
-
-	for user in users:
-		doc = frappe.get_doc("User", user.get("name"))
-		to_remove = [ row for row in doc.user_emails if row.email_account == email_account ]
-		[ doc.remove(row) for row in to_remove ]
-
-		doc.save(ignore_permissions=True)
-
 def ask_pass_update():
 	# update the sys defaults as to awaiting users
 	from frappe.utils import set_default
@@ -809,24 +717,19 @@ def ask_pass_update():
 
 def _get_user_for_update_password(key, old_password):
 	# verify old password
+	result = frappe._dict()
 	if key:
-		user = frappe.db.get_value("User", {"reset_password_key": key})
-		if not user:
-			return {
-				'message': _("The Link specified has either been used before or Invalid")
-			}
+		result.user = frappe.db.get_value("User", {"reset_password_key": key})
+		if not result.user:
+			result.message = _("The Link specified has either been used before or Invalid")
 
 	elif old_password:
 		# verify old password
 		frappe.local.login_manager.check_password(frappe.session.user, old_password)
 		user = frappe.session.user
+		result.user = user
 
-	else:
-		return
-
-	return {
-		'user': user
-	}
+	return result
 
 def reset_user_data(user):
 	user_doc = frappe.get_doc("User", user)
@@ -848,14 +751,12 @@ def sign_up(email, full_name, redirect_to):
 
 	user = frappe.db.get("User", {"email": email})
 	if user:
-		if user.disabled:
-			return 0, _("Registered but disabled")
-		else:
+		if user.enabled:
 			return 0, _("Already Registered")
+		else:
+			return 0, _("Registered but disabled")
 	else:
-		if frappe.db.sql("""select count(*) from tabUser where
-			HOUR(TIMEDIFF(CURRENT_TIMESTAMP, TIMESTAMP(modified)))=1""")[0][0] > 300:
-
+		if frappe.db.get_creation_count('User', 60) > 300:
 			frappe.respond_as_web_page(_('Temporarily Disabled'),
 				_('Too many users signed up recently, so the registration is disabled. Please try back in an hour'),
 				http_status_code=429)
@@ -1048,91 +949,6 @@ def update_gravatar(name):
 	if gravatar:
 		frappe.db.set_value('User', name, 'user_image', gravatar)
 
-@frappe.whitelist(allow_guest=True)
-def send_token_via_sms(tmp_id,phone_no=None,user=None):
-	try:
-		from frappe.core.doctype.sms_settings.sms_settings import send_request
-	except:
-		return False
-
-	if not frappe.cache().ttl(tmp_id + '_token'):
-		return False
-	ss = frappe.get_doc('SMS Settings', 'SMS Settings')
-	if not ss.sms_gateway_url:
-		return False
-
-	token = frappe.cache().get(tmp_id + '_token')
-	args = {ss.message_parameter: 'verification code is {}'.format(token)}
-
-	for d in ss.get("parameters"):
-		args[d.parameter] = d.value
-
-	if user:
-		user_phone = frappe.db.get_value('User', user, ['phone','mobile_no'], as_dict=1)
-		usr_phone = user_phone.mobile_no or user_phone.phone
-		if not usr_phone:
-			return False
-	else:
-		if phone_no:
-			usr_phone = phone_no
-		else:
-			return False
-
-	args[ss.receiver_parameter] = usr_phone
-	status = send_request(ss.sms_gateway_url, args, use_post=ss.use_post)
-
-	if 200 <= status < 300:
-		frappe.cache().delete(tmp_id + '_token')
-		return True
-	else:
-		return False
-
-@frappe.whitelist(allow_guest=True)
-def send_token_via_email(tmp_id,token=None):
-	import pyotp
-
-	user = frappe.cache().get(tmp_id + '_user')
-	count = token or frappe.cache().get(tmp_id + '_token')
-
-	if ((not user) or (user == 'None') or (not count)):
-		return False
-	user_email = frappe.db.get_value('User',user, 'email')
-	if not user_email:
-		return False
-
-	otpsecret = frappe.cache().get(tmp_id + '_otp_secret')
-	hotp = pyotp.HOTP(otpsecret)
-
-	frappe.sendmail(
-		recipients=user_email,
-		sender=None,
-		subject="Verification Code",
-		template="verification_code",
-		args=dict(code=hotp.at(int(count))),
-		delayed=False,
-		retry=3
-	)
-
-	return True
-
-@frappe.whitelist(allow_guest=True)
-def reset_otp_secret(user):
-	otp_issuer = frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name')
-	user_email = frappe.db.get_value('User',user, 'email')
-	if frappe.session.user in ["Administrator", user] :
-		frappe.defaults.clear_default(user + '_otplogin')
-		frappe.defaults.clear_default(user + '_otpsecret')
-		email_args = {
-			'recipients':user_email, 'sender':None, 'subject':'OTP Secret Reset - {}'.format(otp_issuer or "Frappe Framework"),
-			'message':'<p>Your OTP secret on {} has been reset. If you did not perform this reset and did not request it, please contact your System Administrator immediately.</p>'.format(otp_issuer or "Frappe Framework"),
-			'delayed':False,
-			'retry':3
-		}
-		enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, is_async=True, job_name=None, now=False, **email_args)
-		return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login."))
-	else:
-		return frappe.throw(_("OTP secret can only be reset by the Administrator."))
-
 def throttle_user_creation():
 	if frappe.flags.in_import:
 		return
@@ -1150,15 +966,6 @@ def get_module_profile(module_profile):
 	module_profile = frappe.get_doc('Module Profile', {'module_profile_name': module_profile})
 	return module_profile.get('block_modules')
 
-def update_roles(role_profile):
-	users = frappe.get_all('User', filters={'role_profile_name': role_profile})
-	role_profile = frappe.get_doc('Role Profile', role_profile)
-	roles = [role.role for role in role_profile.roles]
-	for d in users:
-		user = frappe.get_doc('User', d)
-		user.set('roles', [])
-		user.add_roles(*roles)
-
 def create_contact(user, ignore_links=False, ignore_mandatory=False):
 	from frappe.contacts.doctype.contact.contact import get_contact_name
 	if user.name in ["Administrator", "Guest"]: return
@@ -1217,18 +1024,18 @@ def generate_keys(user):
 
 	:param user: str
 	"""
-	if "System Manager" in frappe.get_roles():
-		user_details = frappe.get_doc("User", user)
-		api_secret = frappe.generate_hash(length=15)
-		# if api key is not set generate api key
-		if not user_details.api_key:
-			api_key = frappe.generate_hash(length=15)
-			user_details.api_key = api_key
-		user_details.api_secret = api_secret
-		user_details.save()
+	frappe.only_for("System Manager")
+	user_details = frappe.get_doc("User", user)
+	api_secret = frappe.generate_hash(length=15)
+	# if api key is not set generate api key
+	if not user_details.api_key:
+		api_key = frappe.generate_hash(length=15)
+		user_details.api_key = api_key
+	user_details.api_secret = api_secret
+	user_details.save()
+
+	return {"api_secret": api_secret}
 
-		return {"api_secret": api_secret}
-	frappe.throw(frappe._("Not Permitted"), frappe.PermissionError)
 
 @frappe.whitelist()
 def switch_theme(theme):

frappe/desk/doctype/event/test_event.py

@@ -14,7 +14,7 @@ test_records = frappe.get_test_records('Event')
 
 class TestEvent(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql('delete from tabEvent')
+		frappe.db.delete("Event")
 		make_test_objects('Event', reset=True)
 
 		self.test_records = frappe.get_test_records('Event')

frappe/desk/doctype/note/test_note.py

@@ -8,9 +8,9 @@ test_records = frappe.get_test_records('Note')
 
 class TestNote(unittest.TestCase):
 	def insert_note(self):
-		frappe.db.sql('delete from tabVersion')
-		frappe.db.sql('delete from tabNote')
-		frappe.db.sql('delete from `tabNote Seen By`')
+		frappe.db.delete("Version")
+		frappe.db.delete("Note")
+		frappe.db.delete("Note Seen By")
 
 		return frappe.get_doc(dict(doctype='Note', title='test note',
 			content='test note content')).insert()

frappe/desk/doctype/notification_log/test_notification_log.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # See license.txt
 import frappe
+from frappe.core.doctype.user.user import get_system_users
 from frappe.desk.form.assign_to import add as assign_task
 import unittest
 
@@ -54,7 +55,4 @@ def get_todo():
 	return frappe.get_cached_doc('ToDo', res[0].name)
 
 def get_user():
-	users = frappe.db.get_all('User',
-		filters={'name': ('not in', ['Administrator', 'Guest'])},
-		fields='name', limit=1)
-	return users[0].name
+	return get_system_users(limit=1)[0]

frappe/desk/doctype/system_console/system_console.js

@@ -5,7 +5,7 @@ frappe.ui.form.on('System Console', {
 	onload: function(frm) {
 		frappe.ui.keys.add_shortcut({
 			shortcut: 'shift+enter',
-			action: () => frm.execute_action('Execute'),
+			action: () => frm.page.btn_primary.trigger('click'),
 			page: frm.page,
 			description: __('Execute Console script'),
 			ignore_inputs: true,
@@ -14,8 +14,11 @@ frappe.ui.form.on('System Console', {
 
 	refresh: function(frm) {
 		frm.disable_save();
-		frm.page.set_primary_action(__("Execute"), () => {
-			frm.execute_action('Execute');
+		frm.page.set_primary_action(__("Execute"), $btn => {
+			$btn.text(__('Executing...'));
+			return frm.execute_action("Execute").then(() => {
+				$btn.text(__('Execute'));
+			});
 		});
 	}
 });

frappe/desk/doctype/tag/test_tag.py

@@ -6,7 +6,7 @@ from frappe.desk.doctype.tag.tag import add_tag
 
 class TestTag(unittest.TestCase):
 	def setUp(self) -> None:
-		frappe.db.sql("DELETE from `tabTag`")
+		frappe.db.delete("Tag")
 		frappe.db.sql("UPDATE `tabDocType` set _user_tags=''")
 
 	def test_tag_count_query(self):

frappe/desk/doctype/todo/test_todo.py

@@ -14,7 +14,7 @@ class TestToDo(unittest.TestCase):
 		todo = frappe.get_doc(dict(doctype='ToDo', description='test todo',
 			assigned_by='Administrator')).insert()
 
-		frappe.db.sql('delete from `tabDeleted Document`')
+		frappe.db.delete("Deleted Document")
 		todo.delete()
 
 		deleted = frappe.get_doc('Deleted Document', dict(deleted_doctype=todo.doctype, deleted_name=todo.name))
@@ -27,7 +27,7 @@ class TestToDo(unittest.TestCase):
 			frappe.db.get_value('User', todo.assigned_by, 'full_name'))
 
 	def test_fetch_setup(self):
-		frappe.db.sql('delete from tabToDo')
+		frappe.db.delete("ToDo")
 
 		todo_meta = frappe.get_doc('DocType', 'ToDo')
 		todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_from = ''
@@ -104,8 +104,8 @@ class TestToDo(unittest.TestCase):
 		clear_permissions_cache('ToDo')
 		frappe.db.rollback()
 
-def test_fetch_if_empty(self):
-		frappe.db.sql('delete from tabToDo')
+	def test_fetch_if_empty(self):
+		frappe.db.delete("ToDo")
 
 		# Allow user changes
 		todo_meta = frappe.get_doc('DocType', 'ToDo')
@@ -122,9 +122,8 @@ def test_fetch_if_empty(self):
 		self.assertEqual(todo.assigned_by_full_name, 'Admin')
 
 		# Overwrite user changes
-		todo_meta = frappe.get_doc('DocType', 'ToDo')
-		todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_if_empty = 0
-		todo_meta.save()
+		todo.meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_if_empty = 0
+		todo.meta.save()
 
 		todo.reload()
 		todo.save()

frappe/desk/doctype/todo/todo.py

@@ -29,8 +29,15 @@ class ToDo(Document):
 		else:
 			# NOTE the previous value is only available in validate method
 			if self.get_db_value("status") != self.status:
+				if self.owner == frappe.session.user:
+					removal_message = frappe._("{0} removed their assignment.").format(
+						get_fullname(frappe.session.user))
+				else:
+					removal_message = frappe._("Assignment of {0} removed by {1}").format(
+						get_fullname(self.owner), get_fullname(frappe.session.user))
+
 				self._assignment = {
-					"text": frappe._("Assignment closed by {0}").format(get_fullname(frappe.session.user)),
+					"text": removal_message,
 					"comment_type": "Assignment Completed"
 				}
 

frappe/desk/doctype/workspace/workspace.json

@@ -28,7 +28,6 @@
   "pin_to_bottom",
   "hide_custom",
   "public",
-  "content_section",
   "content",
   "section_break_2",
   "charts_label",
@@ -39,6 +38,7 @@
   "section_break_18",
   "cards_label",
   "links",
+  "roles_section",
   "roles"
  ],
  "fields": [
@@ -46,6 +46,7 @@
    "fieldname": "label",
    "fieldtype": "Data",
    "label": "Name",
+   "reqd": 1,
    "unique": 1
   },
   {
@@ -232,21 +233,18 @@
   {
    "fieldname": "title",
    "fieldtype": "Data",
-   "label": "Title"
+   "label": "Title",
+   "reqd": 1
   },
   {
    "fieldname": "parent_page",
    "fieldtype": "Data",
    "label": "Parent Page"
   },
-  {
-   "fieldname": "content_section",
-   "fieldtype": "Section Break",
-   "label": "Content"
-  },
   {
    "fieldname": "content",
    "fieldtype": "Long Text",
+   "hidden": 1,
    "label": "Content"
   },
   {
@@ -259,10 +257,15 @@
    "fieldtype": "Table",
    "label": "Roles",
    "options": "Has Role"
+  },
+  {
+   "fieldname": "roles_section",
+   "fieldtype": "Section Break",
+   "label": "Roles"
   }
  ],
  "links": [],
- "modified": "2021-08-05 11:49:09.028243",
+ "modified": "2021-08-19 12:51:00.233017",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace",

frappe/desk/doctype/workspace/workspace.py

@@ -17,6 +17,12 @@ class Workspace(Document):
 			frappe.throw(_("You need to be in developer mode to edit this document"))
 		validate_route_conflict(self.doctype, self.name)
 
+		try:
+			if not isinstance(loads(self.content), list):
+				raise
+		except Exception:
+			frappe.throw(_("Content data shoud be a list"))
+
 		duplicate_exists = frappe.db.exists("Workspace", {
 			"name": ["!=", self.name], 'is_default': 1, 'extends': self.extends
 		})

frappe/desk/form/utils.py

@@ -5,7 +5,7 @@ import frappe, json
 import frappe.desk.form.meta
 import frappe.desk.form.load
 from frappe.desk.form.document_follow import follow_document
-from frappe.utils.file_manager import extract_images_from_html
+from frappe.core.doctype.file.file import extract_images_from_html
 
 from frappe import _
 

frappe/email/doctype/email_account/email_account.py

@@ -137,8 +137,6 @@ class EmailAccount(Document):
 
 	def on_update(self):
 		"""Check there is only one default of each type."""
-		from frappe.core.doctype.user.user import setup_user_email_inbox
-
 		self.check_automatic_linking_email_account()
 		self.there_must_be_only_one_default()
 		setup_user_email_inbox(email_account=self.name, awaiting_password=self.awaiting_password,
@@ -532,8 +530,6 @@ class EmailAccount(Document):
 
 	def on_trash(self):
 		"""Clear communications where email account is linked"""
-		from frappe.core.doctype.user.user import remove_user_email_inbox
-
 		frappe.db.sql("update `tabCommunication` set email_account='' where email_account=%s", self.name)
 		remove_user_email_inbox(email_account=self.name)
 
@@ -724,3 +720,84 @@ def get_max_email_uid(email_account):
 	else:
 		max_uid = cint(result[0].get("uid", 0)) + 1
 		return max_uid
+
+
+def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing):
+	""" setup email inbox for user """
+	from frappe.core.doctype.user.user import ask_pass_update
+
+	def add_user_email(user):
+		user = frappe.get_doc("User", user)
+		row = user.append("user_emails", {})
+
+		row.email_id = email_id
+		row.email_account = email_account
+		row.awaiting_password = awaiting_password or 0
+		row.enable_outgoing = enable_outgoing or 0
+
+		user.save(ignore_permissions=True)
+
+	update_user_email_settings = False
+	if not all([email_account, email_id]):
+		return
+
+	user_names = frappe.db.get_values("User", {"email": email_id}, as_dict=True)
+	if not user_names:
+		return
+
+	for user in user_names:
+		user_name = user.get("name")
+
+		# check if inbox is alreay configured
+		user_inbox = frappe.db.get_value("User Email", {
+			"email_account": email_account,
+			"parent": user_name
+		}, ["name"]) or None
+
+		if not user_inbox:
+			add_user_email(user_name)
+		else:
+			# update awaiting password for email account
+			update_user_email_settings = True
+
+	if update_user_email_settings:
+		frappe.db.sql("""UPDATE `tabUser Email` SET awaiting_password = %(awaiting_password)s,
+			enable_outgoing = %(enable_outgoing)s WHERE email_account = %(email_account)s""", {
+				"email_account": email_account,
+				"enable_outgoing": enable_outgoing,
+				"awaiting_password": awaiting_password or 0
+			})
+	else:
+		users = " and ".join([frappe.bold(user.get("name")) for user in user_names])
+		frappe.msgprint(_("Enabled email inbox for user {0}").format(users))
+	ask_pass_update()
+
+def remove_user_email_inbox(email_account):
+	""" remove user email inbox settings if email account is deleted """
+	if not email_account:
+		return
+
+	users = frappe.get_all("User Email", filters={
+		"email_account": email_account
+	}, fields=["parent as name"])
+
+	for user in users:
+		doc = frappe.get_doc("User", user.get("name"))
+		to_remove = [row for row in doc.user_emails if row.email_account == email_account]
+		[doc.remove(row) for row in to_remove]
+
+		doc.save(ignore_permissions=True)
+
+@frappe.whitelist(allow_guest=False)
+def set_email_password(email_account, user, password):
+	account = frappe.get_doc("Email Account", email_account)
+	if account.awaiting_password:
+		account.awaiting_password = 0
+		account.password = password
+		try:
+			account.save(ignore_permissions=True)
+		except Exception:
+			frappe.db.rollback()
+			return False
+
+	return True

frappe/email/doctype/email_account/test_email_account.py

@@ -34,8 +34,8 @@ class TestEmailAccount(unittest.TestCase):
 	def setUp(self):
 		frappe.flags.mute_emails = False
 		frappe.flags.sent_mail = None
-		frappe.db.sql('delete from `tabEmail Queue`')
-		frappe.db.sql('delete from `tabUnhandled Email`')
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Unhandled Email")
 
 	def get_test_mail(self, fname):
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:
@@ -60,7 +60,7 @@ class TestEmailAccount(unittest.TestCase):
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		comm.db_set("creation", datetime.now() - timedelta(seconds = 30 * 60))
 
-		frappe.db.sql("DELETE FROM `tabEmail Queue`")
+		frappe.db.delete("Email Queue")
 		notify_unreplied()
 		self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype,
 			"reference_name": comm.reference_name, "status":"Not Sent"}))
@@ -183,7 +183,7 @@ class TestEmailAccount(unittest.TestCase):
 
 	def test_threading_by_message_id(self):
 		cleanup()
-		frappe.db.sql("""delete from `tabEmail Queue`""")
+		frappe.db.delete("Email Queue")
 
 		# reference document for testing
 		event = frappe.get_doc(dict(doctype='Event', subject='test-message')).insert()
@@ -242,8 +242,8 @@ class TestInboundMail(unittest.TestCase):
 
 	def setUp(self):
 		cleanup()
-		frappe.db.sql('delete from `tabEmail Queue`')
-		frappe.db.sql('delete from `tabToDo`')
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("ToDo")
 
 	def get_test_mail(self, fname):
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:

frappe/email/doctype/newsletter/newsletter.py

@@ -165,6 +165,7 @@ class Newsletter(WebsiteGenerator):
 		sender = self.send_from or frappe.utils.get_formatted_email(self.owner)
 		args = {"message": message, "name": self.name}
 
+		is_auto_commit_set = bool(frappe.db.auto_commit_on_many_writes)
 		frappe.db.auto_commit_on_many_writes = not frappe.flags.in_test
 
 		frappe.sendmail(
@@ -184,7 +185,7 @@ class Newsletter(WebsiteGenerator):
 			args=args,
 		)
 
-		frappe.db.auto_commit_on_many_writes = not frappe.flags.in_test
+		frappe.db.auto_commit_on_many_writes = is_auto_commit_set
 
 	def get_message(self) -> str:
 		if self.content_type == "HTML":

frappe/email/doctype/notification/test_notification.py

@@ -9,7 +9,7 @@ test_dependencies = ["User", "Notification"]
 
 class TestNotification(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql("""delete from `tabEmail Queue`""")
+		frappe.db.delete("Email Queue")
 		frappe.set_user("test@example.com")
 
 		if not frappe.db.exists('Notification', {'name': 'ToDo Status Update'}, 'name'):
@@ -50,7 +50,7 @@ class TestNotification(unittest.TestCase):
 
 		self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "Communication",
 			"reference_name": communication.name, "status":"Not Sent"}))
-		frappe.db.sql("""delete from `tabEmail Queue`""")
+		frappe.db.delete("Email Queue")
 
 		communication.reload()
 		communication.content = "test 2"
@@ -189,9 +189,9 @@ class TestNotification(unittest.TestCase):
 
 	def test_cc_jinja(self):
 
-		frappe.db.sql("""delete from `tabUser` where email='test_jinja@example.com'""")
-		frappe.db.sql("""delete from `tabEmail Queue`""")
-		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
+		frappe.db.delete("User", {"email": "test_jinja@example.com"})
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Email Queue Recipient")
 
 		test_user = frappe.new_doc("User")
 		test_user.name = 'test_jinja'
@@ -205,9 +205,9 @@ class TestNotification(unittest.TestCase):
 
 		self.assertTrue(frappe.db.get_value("Email Queue Recipient", {"recipient": "test_jinja@example.com"}))
 
-		frappe.db.sql("""delete from `tabUser` where email='test_jinja@example.com'""")
-		frappe.db.sql("""delete from `tabEmail Queue`""")
-		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
+		frappe.db.delete("User", {"email": "test_jinja@example.com"})
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Email Queue Recipient")
 
 	def test_notification_to_assignee(self):
 		todo = frappe.new_doc('ToDo')

frappe/integrations/doctype/webhook/test_webhook.py

@@ -11,9 +11,9 @@ class TestWebhook(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
 		# delete any existing webhooks
-		frappe.db.sql("DELETE FROM tabWebhook")
+		frappe.db.delete("Webhook")
 		# Delete existing logs if any
-		frappe.db.sql("DELETE FROM `tabWebhook Request Log`")
+		frappe.db.delete("Webhook Request Log")
 		# create test webhooks
 		cls.create_sample_webhooks()
 
@@ -46,7 +46,7 @@ class TestWebhook(unittest.TestCase):
 	@classmethod
 	def tearDownClass(cls):
 		# delete any existing webhooks
-		frappe.db.sql("DELETE FROM tabWebhook")
+		frappe.db.delete("Webhook")
 
 	def setUp(self):
 		# retrieve or create a User webhook for `after_insert`
@@ -168,7 +168,7 @@ class TestWebhook(unittest.TestCase):
 	def test_webhook_req_log_creation(self):
 		if not frappe.db.get_value('User', 'user2@integration.webhooks.test.com'):
 			user = frappe.get_doc({
-				'doctype': 'User', 
+				'doctype': 'User',
 				'email': 'user2@integration.webhooks.test.com',
 				'first_name': 'user2'
 			}).insert()

frappe/model/delete_doc.py

@@ -10,7 +10,7 @@ import frappe.model.meta
 from frappe import _
 from frappe import get_module_path
 from frappe.model.dynamic_links import get_dynamic_link_map
-from frappe.core.doctype.file.file import remove_all
+from frappe.utils.file_manager import remove_all
 from frappe.utils.password import delete_all_passwords_for
 from frappe.model.naming import revert_series_if_last
 from frappe.utils.global_search import delete_for_document
@@ -190,7 +190,7 @@ def delete_from_table(doctype, name, ignore_doctypes, doc):
 	# delete from child tables
 	for t in list(set(tables)):
 		if t not in ignore_doctypes:
-			frappe.db.sql("delete from `tab%s` where parenttype=%s and parent = %s" % (t, '%s', '%s'), (doctype, name))
+			frappe.db.delete(t, {"parenttype": doctype, "parent": name})
 
 def update_flags(doc, flags=None, ignore_permissions=False):
 	if ignore_permissions:
@@ -323,9 +323,10 @@ def delete_dynamic_links(doctype, name):
 
 def delete_references(doctype, reference_doctype, reference_name,
 		reference_doctype_field = 'reference_doctype', reference_name_field = 'reference_name'):
-	frappe.db.sql('''delete from `tab{0}`
-		where {1}=%s and {2}=%s'''.format(doctype, reference_doctype_field, reference_name_field), # nosec
-		(reference_doctype, reference_name))
+	frappe.db.delete(doctype, {
+		reference_doctype_field: reference_doctype,
+		reference_name_field: reference_name
+	})
 
 def clear_references(doctype, reference_doctype, reference_name,
 		reference_doctype_field = 'reference_doctype', reference_name_field = 'reference_name'):

frappe/model/document.py

@@ -385,8 +385,7 @@ class Document(BaseDocument):
 					[self.name, self.doctype, fieldname] + rows)
 			if len(deleted_rows) > 0:
 				# delete rows that do not match the ones in the document
-				frappe.db.sql("""delete from `tab{0}` where name in ({1})""".format(df.options,
-					','.join(['%s'] * len(deleted_rows))), tuple(row[0] for row in deleted_rows))
+				frappe.db.delete(df.options, {"name": ("in", tuple(row[0] for row in deleted_rows))})
 
 		else:
 			# no rows found, delete all rows

frappe/modules/utils.py

@@ -114,8 +114,7 @@ def sync_customizations_for_doctype(data, folder):
 					doc.db_insert()
 
 			if custom_doctype != 'Custom Field':
-				frappe.db.sql('delete from `tab{0}` where `{1}` =%s'.format(
-					custom_doctype, doctype_fieldname), doc_type)
+				frappe.db.delete(custom_doctype, {doctype_fieldname: doc_type})
 
 				for d in data[key]:
 					_insert(d)

frappe/patches.txt

@@ -182,4 +182,4 @@ frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
 frappe.patches.v14_0.drop_data_import_legacy
 frappe.patches.v14_0.rename_cancelled_documents
-frappe.patches.v14_0.update_workspace2
+frappe.patches.v14_0.update_workspace2 # 25.08.2021

frappe/patches/v14_0/update_workspace2.py

@@ -50,11 +50,11 @@ def create_content(doc):
 	return content
 
 def update_wspace(doc, seq, content):
-	if not doc.is_standard and not doc.public:
+	if not doc.title and not doc.content and not doc.is_standard and not doc.public:
 		doc.sequence_id = seq + 1
 		doc.content = json.dumps(content)
 		doc.public = 0
-		doc.title = doc.extends
+		doc.title = doc.extends or doc.label
 		doc.extends = ''
 		doc.category = ''
 		doc.onboarding = ''

frappe/public/js/frappe/desk.js

@@ -230,7 +230,7 @@ frappe.Application = class Application {
 			s.fields_dict.checking.$wrapper.html('<i class="fa fa-spinner fa-spin fa-4x"></i>');
 			s.show();
 			frappe.call({
-				method: 'frappe.core.doctype.user.user.set_email_password',
+				method: 'frappe.email.doctype.email_account.email_account.set_email_password',
 				args: {
 					"email_account": email_account[i]["email_account"],
 					"user": user,

frappe/public/js/frappe/form/controls/data.js

@@ -163,7 +163,7 @@ frappe.ui.form.ControlData = class ControlData extends frappe.ui.form.ControlInp
 			}
 		};
 		this.$input.on("change", change_handler);
-		if (this.constructor.trigger_change_on_input_event) {
+		if (this.constructor.trigger_change_on_input_event && !this.in_grid()) {
 			// debounce to avoid repeated validations on value change
 			this.$input.on("input", frappe.utils.debounce(change_handler, 500));
 		}
@@ -267,4 +267,7 @@ frappe.ui.form.ControlData = class ControlData extends frappe.ui.form.ControlInp
 		let el = this.$input.parents(el_class)[0];
 		if (el) $(el).toggleClass(scroll_class, add);
 	}
+	in_grid() {
+		return this.grid || this.layout && this.layout.grid;
+	}
 };

frappe/public/js/frappe/form/form.js

@@ -339,7 +339,7 @@ frappe.ui.form.Form = class FrappeForm {
 			}
 		}
 		if (action.action_type==='Server Action') {
-			frappe.xcall(action.action, {'doc': this.doc}).then((doc) => {
+			return frappe.xcall(action.action, {'doc': this.doc}).then((doc) => {
 				if (doc.doctype) {
 					// document is returned by the method,
 					// apply the changes locally and refresh
@@ -354,7 +354,7 @@ frappe.ui.form.Form = class FrappeForm {
 				});
 			});
 		} else if (action.action_type==='Route') {
-			frappe.set_route(action.action);
+			return frappe.set_route(action.action);
 		}
 	}
 

frappe/public/js/frappe/form/form_tour.js

@@ -82,10 +82,16 @@ frappe.ui.form.FormTour = class FormTour {
 
 	get_step(step_info, on_next) {
 		const { name, fieldname, title, description, position, is_table_field } = step_info;
+		let element = `.frappe-control[data-fieldname='${fieldname}']`;
+
 		const field = this.frm.get_field(fieldname);
-		let element = field ? field.wrapper : `.frappe-control[data-fieldname='${fieldname}']`;
+		if (field) {
+			// wrapper for section breaks returns in a list
+			element = field.wrapper[0] ? field.wrapper[0] : field.wrapper;
+		}
 
 		if (is_table_field) {
+			// TODO: fix wrapper for grid sections
 			element = `.grid-row-open .frappe-control[data-fieldname='${fieldname}']`;
 		}
 

frappe/public/js/frappe/form/grid.js

@@ -38,7 +38,7 @@ export default class Grid {
 
 		this.is_grid = true;
 		this.debounced_refresh = this.refresh.bind(this);
-		this.debounced_refresh = frappe.utils.debounce(this.debounced_refresh, 500);
+		this.debounced_refresh = frappe.utils.debounce(this.debounced_refresh, 100);
 	}
 
 	allow_on_grid_editing() {

frappe/public/js/frappe/form/layout.js

@@ -252,14 +252,18 @@ frappe.ui.form.Layout = class Layout {
 		}
 
 		if (document.activeElement) {
-			document.activeElement.focus();
-	
-			if (document.activeElement.tagName == 'INPUT') {
+			if (document.activeElement.tagName == 'INPUT' && this.is_numeric_field_active()) {
 				document.activeElement.select();
 			}
 		}
 	}
 
+	is_numeric_field_active() {
+		const control = $(document.activeElement).closest(".frappe-control");
+		const fieldtype = (control.data() || {}).fieldtype;
+		return frappe.model.numeric_fieldtypes.includes(fieldtype);
+	}
+
 	refresh_sections() {
 		// hide invisible sections
 		this.wrapper.find(".form-section:not(.hide-control)").each(function() {

frappe/public/js/frappe/router.js

@@ -129,7 +129,7 @@ frappe.router = {
 		if (frappe.workspaces[route[0]]) {
 			// public workspace
 			route = ['Workspaces', frappe.workspaces[route[0]].title];
-		} else if (frappe.workspaces[route[1]]) {
+		} else if (route[0] == 'private' && frappe.workspaces[route[1]]) {
 			// private workspace
 			route = ['Workspaces', 'private', frappe.workspaces[route[1]].title];
 		} else if (this.routes[route[0]]) {
@@ -354,8 +354,8 @@ frappe.router = {
 				return a;
 			}
 		}).join('/');
-
-		return '/app/' + (path_string || 'home');
+		let default_page = frappe.workspaces['home'] ? 'home' : Object.keys(frappe.workspaces)[0];
+		return '/app/' + (path_string || default_page);
 	},
 
 	push_state(url) {

frappe/public/js/frappe/ui/page.js

@@ -251,7 +251,7 @@ frappe.ui.Page = class Page {
 			.prop("disabled", false)
 			.html(opts.label)
 			.on("click", function() {
-				let response = opts.click.apply(this);
+				let response = opts.click.apply(this, [btn]);
 				me.btn_disable_enable(btn, response);
 			});
 

frappe/public/js/frappe/ui/toolbar/awesome_bar.js

@@ -194,9 +194,14 @@ frappe.search.AwesomeBar = class AwesomeBar {
 		var out = [], routes = [];
 		options.forEach(function(option) {
 			if(option.route) {
-				if(option.route[0] === "List" && option.route[2] !== 'Report') {
+				if (
+					option.route[0] === "List" &&
+					option.route[2] !== 'Report' &&
+					option.route[2] !== 'Inbox'
+				) {
 					option.route.splice(2);
 				}
+
 				var str_route = (typeof option.route==='string') ?
 					option.route : option.route.join('/');
 				if(routes.indexOf(str_route)===-1) {

frappe/public/js/frappe/views/inbox/inbox_view.js

@@ -94,19 +94,20 @@ frappe.views.InboxView = class InboxView extends frappe.views.ListView {
 		this.render_list();
 		this.on_row_checked();
 		this.render_count();
-		this.render_tags();
 	}
 
 	get_meta_html(email) {
 		const attachment = email.has_attachment ?
 			`<span class="fa fa-paperclip fa-large" title="${__('Has Attachments')}"></span>` : '';
 
-		const form_link = frappe.utils.get_form_link(email.reference_doctype, email.reference_name);
-		const link = email.reference_doctype && email.reference_doctype !== this.doctype ?
-			`<a class="text-muted grey" href="${form_link}"
+		let link = "";
+		if (email.reference_doctype && email.reference_doctype !== this.doctype) {
+			link = `<a class="text-muted grey"
+				href="${frappe.utils.get_form_link(email.reference_doctype, email.reference_name)}"
 				title="${__('Linked with {0}', [email.reference_doctype])}">
 				<i class="fa fa-link fa-large"></i>
-			</a>` : '';
+			</a>`;
+		}
 
 		const communication_date = comment_when(email.communication_date, true);
 		const status =

frappe/public/js/frappe/views/workspace/workspace.js

@@ -174,10 +174,6 @@ frappe.views.Workspace = class Workspace {
 			$(e.target).parent().find('.sidebar-item-container').toggleClass('hidden');
 		});
 
-		if (!this.current_page.name) {
-			$title.trigger("click");
-		}
-
 		if (Object.keys(root_pages).length === 0) {
 			sidebar_section.addClass('hidden');
 		}

frappe/rate_limiter.py

@@ -103,7 +103,7 @@ def rate_limit(key: str, limit: Union[int, Callable] = 5, seconds: int= 24*60*60
 		def wrapper(*args, **kwargs):
 			# Do not apply rate limits if method is not opted to check
 			if methods != 'ALL' and frappe.request.method.upper() not in methods:
-				return frappe.call(fun, **frappe.form_dict)
+				return frappe.call(fun, **frappe.form_dict or kwargs)
 
 			_limit = limit() if callable(limit) else limit
 
@@ -118,6 +118,6 @@ def rate_limit(key: str, limit: Union[int, Callable] = 5, seconds: int= 24*60*60
 			if value > _limit:
 				frappe.throw(_("You hit the rate limit because of too many requests. Please try after sometime."))
 
-			return frappe.call(fun, **frappe.form_dict)
+			return frappe.call(fun, **frappe.form_dict or kwargs)
 		return wrapper
 	return ratelimit_decorator

frappe/social/doctype/energy_point_log/test_energy_point_log.py

@@ -13,8 +13,8 @@ class TestEnergyPointLog(unittest.TestCase):
 
 	def tearDown(self):
 		frappe.set_user('Administrator')
-		frappe.db.sql('DELETE FROM `tabEnergy Point Log`')
-		frappe.db.sql('DELETE FROM `tabEnergy Point Rule`')
+		frappe.db.delete("Energy Point Log")
+		frappe.db.delete("Energy Point Rule")
 		frappe.cache().delete_value('energy_point_rule_map')
 
 	def test_user_energy_point(self):

frappe/tests/test_assign.py

@@ -22,7 +22,7 @@ class TestAssign(unittest.TestCase):
 		self.assertEqual(len(assignments), 0)
 
 	def test_assignment_count(self):
-		frappe.db.sql('delete from tabToDo')
+		frappe.db.delete("ToDo")
 
 		if not frappe.db.exists("User", "test_assign1@example.com"):
 			frappe.get_doc({"doctype":"User", "email":"test_assign1@example.com", "first_name":"Test", "roles": [{"role": "System Manager"}]}).insert()

frappe/tests/test_db_query.py

@@ -105,7 +105,7 @@ class TestReportview(unittest.TestCase):
 
 	def test_between_filters(self):
 		""" test case to check between filter for date fields """
-		frappe.db.sql("delete from tabEvent")
+		frappe.db.delete("Event")
 
 		# create events to test the between operator filter
 		todays_event = create_event()

frappe/tests/test_domainification.py

@@ -17,9 +17,9 @@ class TestDomainification(unittest.TestCase):
 		self.add_active_domain("_Test Domain 1")
 
 	def tearDown(self):
-		frappe.db.sql("delete from tabRole where name='_Test Role'")
-		frappe.db.sql("delete from `tabHas Role` where role='_Test Role'")
-		frappe.db.sql("delete from tabDomain where name in ('_Test Domain 1', '_Test Domain 2')")
+		frappe.db.delete("Role", {"name": "_Test Role"})
+		frappe.db.delete("Has Role", {"role": "_Test Role"})
+		frappe.db.delete("Domain", {"name": ("in", ("_Test Domain 1", "_Test Domain 2"))})
 		frappe.delete_doc('DocType', 'Test Domainification')
 		self.remove_from_active_domains(remove_all=True)
 

frappe/tests/test_dynamic_links.py

@@ -4,7 +4,7 @@ import frappe, unittest
 
 class TestDynamicLinks(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql('delete from `tabEmail Unsubscribe`')
+		frappe.db.delete("Email Unsubscribe")
 
 	def test_delete_normal(self):
 		event = frappe.get_doc({

frappe/tests/test_email.py

@@ -7,9 +7,9 @@ test_dependencies = ['Email Account']
 
 class TestEmail(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql("""delete from `tabEmail Unsubscribe`""")
-		frappe.db.sql("""delete from `tabEmail Queue`""")
-		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
+		frappe.db.delete("Email Unsubscribe")
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Email Queue Recipient")
 
 	def test_email_queue(self, send_after=None):
 		frappe.sendmail(recipients=['test@example.com', 'test1@example.com'],
@@ -170,7 +170,7 @@ class TestEmail(unittest.TestCase):
 		import re
 		email_account = frappe.get_doc('Email Account', '_Test Email Account 1')
 
-		frappe.db.sql('''delete from `tabCommunication` where sender = 'sukh@yyy.com' ''')
+		frappe.db.delete("Communication", {"sender": "sukh@yyy.com"})
 
 		with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw:
 			mails = email_account.get_inbound_mails(test_mails=[raw.read()])

frappe/tests/test_frappe_client.py

@@ -12,7 +12,7 @@ import base64
 class TestFrappeClient(unittest.TestCase):
 	def test_insert_many(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')")
+		frappe.db.delete("Note", {"title": ("in", ('Sing','a','song','of','sixpence'))})
 		frappe.db.commit()
 
 		server.insert_many([
@@ -31,7 +31,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_create_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'test_create'")
+		frappe.db.delete("Note", {"title": "test_create"})
 		frappe.db.commit()
 
 		server.insert({"doctype": "Note", "public": True, "title": "test_create"})
@@ -46,7 +46,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_get_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'get_this'")
+		frappe.db.delete("Note", {"title": "get_this"})
 		frappe.db.commit()
 
 		server.insert_many([
@@ -57,7 +57,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_get_value(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'get_value'")
+		frappe.db.delete("Note", {"title": "get_value"})
 		frappe.db.commit()
 
 		test_content = "test get value"
@@ -82,7 +82,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_update_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title in ('Sing','sing')")
+		frappe.db.delete("Note", {"title": ("in", ("Sing", "sing"))})
 		frappe.db.commit()
 
 		server.insert({"doctype":"Note", "public": True, "title": "Sing"})
@@ -94,12 +94,12 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_update_child_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabContact` where first_name = 'George' and last_name = 'Steevens'")
-		frappe.db.sql("delete from `tabContact` where first_name = 'William' and last_name = 'Shakespeare'")
-		frappe.db.sql("delete from `tabCommunication` where reference_doctype = 'Event'")
-		frappe.db.sql("delete from `tabCommunication Link` where link_doctype = 'Contact'")
-		frappe.db.sql("delete from `tabEvent` where subject = 'Sing a song of sixpence'")
-		frappe.db.sql("delete from `tabEvent Participants` where reference_doctype = 'Contact'")
+		frappe.db.delete("Contact", {"first_name": "George", "last_name": "Steevens"})
+		frappe.db.delete("Contact", {"first_name": "William", "last_name": "Shakespeare"})
+		frappe.db.delete("Communication", {"reference_doctype": "Event"})
+		frappe.db.delete("Communication Link", {"link_doctype": "Contact"})
+		frappe.db.delete("Event", {"subject": "Sing a song of sixpence"})
+		frappe.db.delete("Event Participants", {"reference_doctype": "Contact"})
 		frappe.db.commit()
 
 		# create multiple contacts
@@ -131,7 +131,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_delete_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'delete'")
+		frappe.db.delete("Note", {"title": "delete"})
 		frappe.db.commit()
 
 		server.insert_many([

frappe/tests/test_global_search.py

@@ -24,15 +24,15 @@ class TestGlobalSearch(unittest.TestCase):
 		make_property_setter(doctype, "repeat_on", "in_global_search", 0, "Int")
 
 	def tearDown(self):
-		frappe.db.sql("DELETE FROM `tabProperty Setter` WHERE `doc_type`='Event'")
+		frappe.db.delete("Property Setter", {"doc_type": "Event"})
 		frappe.clear_cache(doctype='Event')
-		frappe.db.sql('DELETE FROM `tabEvent`')
-		frappe.db.sql('DELETE FROM `__global_search`')
+		frappe.db.delete("Event")
+		frappe.db.delete("__global_search")
 		make_test_objects('Event')
 		frappe.db.commit()
 
 	def insert_test_events(self):
-		frappe.db.sql('DELETE FROM `tabEvent`')
+		frappe.db.delete("Event")
 		phrases = ['"The Sixth Extinction II: Amor Fati" is the second episode of the seventh season of the American science fiction.',
 		'After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. ',
 		'Carter explored themes of extraterrestrial involvement in ancient mass extinctions in this episode, the third in a trilogy.']
@@ -97,7 +97,7 @@ class TestGlobalSearch(unittest.TestCase):
 		self.assertEqual(len(results), 0)
 
 	def test_insert_child_table(self):
-		frappe.db.sql('delete from tabEvent')
+		frappe.db.delete("Event")
 		phrases = ['Hydrus is a small constellation in the deep southern sky. ',
 		'It was first depicted on a celestial atlas by Johann Bayer in his 1603 Uranometria. ',
 		'The French explorer and astronomer Nicolas Louis de Lacaille charted the brighter stars and gave their Bayer designations in 1756. ',

frappe/tests/test_goal.py

@@ -13,7 +13,7 @@ class TestGoal(unittest.TestCase):
 		make_test_objects('Event', reset=True)
 
 	def tearDown(self):
-		frappe.db.sql('delete from `tabEvent`')
+		frappe.db.delete("Event")
 		# make_test_objects('Event', reset=True)
 		frappe.db.commit()
 

frappe/tests/test_naming.py

@@ -72,7 +72,7 @@ class TestNaming(unittest.TestCase):
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 0)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = 'TEST-{}-'.format(year)
 		key = 'TEST-.YYYY.-.#####'
@@ -82,40 +82,40 @@ class TestNaming(unittest.TestCase):
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 1)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = 'TEST-'
 		key = 'TEST-'
 		name = 'TEST-00003'
-		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.delete("Series", {"name": series})
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 2)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = 'TEST1-'
 		key = 'TEST1-.#####.-2021-22'
 		name = 'TEST1-00003-2021-22'
-		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.delete("Series", {"name": series})
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 2)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = ''
 		key = '.#####.-2021-22'
 		name = '00003-2021-22'
-		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.delete("Series", {"name": series})
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 2)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 	def test_naming_for_cancelled_and_amended_doc(self):
 		submittable_doctype = frappe.get_doc({

frappe/tests/test_permissions.py

@@ -38,7 +38,7 @@ class TestPermissions(unittest.TestCase):
 		reset('Blogger')
 		reset('Blog Post')
 
-		frappe.db.sql('delete from `tabUser Permission`')
+		frappe.db.delete("User Permission")
 
 		frappe.set_user("test1@example.com")
 
@@ -334,9 +334,9 @@ class TestPermissions(unittest.TestCase):
 			doctype"""
 
 		frappe.set_user('Administrator')
-		frappe.db.sql('DELETE FROM `tabContact`')
-		frappe.db.sql('DELETE FROM `tabContact Email`')
-		frappe.db.sql('DELETE FROM `tabContact Phone`')
+		frappe.db.delete("Contact")
+		frappe.db.delete("Contact Email")
+		frappe.db.delete("Contact Phone")
 
 		reset('Salutation')
 		reset('Contact')

frappe/tests/test_scheduler.py

@@ -45,7 +45,7 @@ class TestScheduler(TestCase):
 
 		# 1st job is in the queue (or running), don't enqueue it again
 		self.assertFalse(job.enqueue())
-		frappe.db.sql('DELETE FROM `tabScheduled Job Log` WHERE `scheduled_job_type`=%s', job.name)
+		frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": job.name})
 
 	def test_is_dormant(self):
 		self.assertTrue(is_dormant(check_time= get_datetime('2100-01-01 00:00:00')))

frappe/twofactor.py

@@ -398,3 +398,23 @@ def should_remove_barcode_image(barcode):
 
 def disable():
 	frappe.db.set_value('System Settings', None, 'enable_two_factor_auth', 0)
+
+@frappe.whitelist()
+def reset_otp_secret(user):
+	otp_issuer = frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name')
+	user_email = frappe.db.get_value('User', user, 'email')
+	if frappe.session.user in ["Administrator", user] :
+		frappe.defaults.clear_default(user + '_otplogin')
+		frappe.defaults.clear_default(user + '_otpsecret')
+		email_args = {
+			'recipients': user_email,
+			'sender': None,
+			'subject': _('OTP Secret Reset - {0}').format(otp_issuer or "Frappe Framework"),
+			'message': _('<p>Your OTP secret on {0} has been reset. If you did not perform this reset and did not request it, please contact your System Administrator immediately.</p>').format(otp_issuer or "Frappe Framework"),
+			'delayed':False,
+			'retry':3
+		}
+		enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, is_async=True, job_name=None, now=False, **email_args)
+		return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login."))
+	else:
+		return frappe.throw(_("OTP secret can only be reset by the Administrator."))

frappe/utils/csvutils.py

@@ -6,16 +6,7 @@ import json
 import csv
 import requests
 from io import StringIO
-from frappe.utils import encode, cstr, cint, flt, comma_or
-
-def read_csv_content_from_uploaded_file(ignore_encoding=False):
-	if getattr(frappe, "uploaded_file", None):
-		with open(frappe.uploaded_file, "r") as upfile:
-			fcontent = upfile.read()
-	else:
-		_file = frappe.new_doc("File")
-		fcontent = _file.get_uploaded_content()
-	return read_csv_content(fcontent, ignore_encoding)
+from frappe.utils import cstr, cint, flt, comma_or
 
 def read_csv_content_from_attached_file(doc):
 	fileid = frappe.get_all("File", fields = ["name"], filters = {"attached_to_doctype": doc.doctype,

frappe/utils/file_manager.py

@@ -213,28 +213,22 @@ def write_file(content, fname, is_private=0):
 	return get_files_path(fname, is_private=is_private)
 
 
-def remove_all(dt, dn, from_delete=False):
+def remove_all(dt, dn, from_delete=False, delete_permanently=False):
 	"""remove all files in a transaction"""
 	try:
 		for fid in frappe.db.sql_list("""select name from `tabFile` where
 			attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)):
-			remove_file(fid, dt, dn, from_delete)
+			if from_delete:
+				# If deleting a doc, directly delete files
+				frappe.delete_doc("File", fid, ignore_permissions=True, delete_permanently=delete_permanently)
+			else:
+				# Removes file and adds a comment in the document it is attached to
+				remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn,
+					from_delete=from_delete, delete_permanently=delete_permanently)
 	except Exception as e:
 		if e.args[0]!=1054: raise # (temp till for patched)
 
-
-def remove_file_by_url(file_url, doctype=None, name=None):
-	if doctype and name:
-		fid = frappe.db.get_value("File", {"file_url": file_url,
-			"attached_to_doctype": doctype, "attached_to_name": name})
-	else:
-		fid = frappe.db.get_value("File", {"file_url": file_url})
-
-	if fid:
-		return remove_file(fid)
-
-
-def remove_file(fid, attached_to_doctype=None, attached_to_name=None, from_delete=False):
+def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_delete=False, delete_permanently=False):
 	"""Remove file and File entry"""
 	file_name = None
 	if not (attached_to_doctype and attached_to_name):
@@ -252,8 +246,7 @@ def remove_file(fid, attached_to_doctype=None, attached_to_name=None, from_delet
 		if not file_name:
 			file_name = frappe.db.get_value("File", fid, "file_name")
 		comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name))
-
-	frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions)
+		frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions, delete_permanently=delete_permanently)
 
 	return comment
 
@@ -372,76 +365,6 @@ def download_file(file_url):
 	frappe.local.response.filecontent = filedata
 	frappe.local.response.type = "download"
 
-def extract_images_from_doc(doc, fieldname):
-	content = doc.get(fieldname)
-	content = extract_images_from_html(doc, content)
-	if frappe.flags.has_dataurl:
-		doc.set(fieldname, content)
-
-
-def extract_images_from_html(doc, content):
-	frappe.flags.has_dataurl = False
-
-	def _save_file(match):
-		data = match.group(1)
-		data = data.split("data:")[1]
-		headers, content = data.split(",")
-		mtype = headers.split(";")[0]
-
-		if isinstance(content, str):
-			content = content.encode("utf-8")
-		if b"," in content:
-			content = content.split(b",")[1]
-		content = base64.b64decode(content)
-		
-		content = optimize_image(content, mtype)
-
-		if "filename=" in headers:
-			filename = headers.split("filename=")[-1]
-
-			# decode filename
-			if not isinstance(filename, str):
-				filename = str(filename, 'utf-8')
-		else:
-			filename = get_random_filename(content_type=mtype)
-
-		doctype = doc.parenttype if doc.parent else doc.doctype
-		name = doc.parent or doc.name
-
-		if doc.doctype == "Comment":
-			doctype = doc.reference_doctype
-			name = doc.reference_name
-
-		# TODO fix this
-		file_url = save_file(filename, content, doctype, name, decode=False).get("file_url")
-		if not frappe.flags.has_dataurl:
-			frappe.flags.has_dataurl = True
-
-		return '<img src="{file_url}"'.format(file_url=file_url)
-
-	if content:
-		content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
-
-	return content
-
-
-def get_random_filename(extn=None, content_type=None):
-	if extn:
-		if not extn.startswith("."):
-			extn = "." + extn
-
-	elif content_type:
-		extn = mimetypes.guess_extension(content_type)
-
-	return random_string(7) + (extn or "")
-
-@frappe.whitelist(allow_guest=True)
-def validate_filename(filename):
-	from frappe.utils import now_datetime
-	timestamp = now_datetime().strftime(" %Y-%m-%d %H:%M:%S")
-	fname = get_file_name(filename, timestamp)
-	return fname
-
 @frappe.whitelist()
 def add_attachments(doctype, name, attachments):
 	'''Add attachments to the given DocType'''

frappe/website/doctype/personal_data_download_request/personal_data_download_request.py

@@ -28,7 +28,7 @@ class PersonalDataDownloadRequest(Document):
 		})
 		f.save(ignore_permissions=True)
 
-		file_link = frappe.utils.get_url("/api/method/frappe.core.doctype.file.file.download_file") +\
+		file_link = frappe.utils.get_url("/api/method/frappe.utils.file_manager.download_file") +\
 			"?" + get_signed_params({"file_url": f.file_url})
 		host_name = frappe.local.site
 		frappe.sendmail(

frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py

@@ -13,7 +13,7 @@ class TestRequestPersonalData(unittest.TestCase):
 		create_user_if_not_exists(email='test_privacy@example.com')
 
 	def tearDown(self):
-		frappe.db.sql("""DELETE FROM `tabPersonal Data Download Request`""")
+		frappe.db.delete("Personal Data Download Request")
 
 	def test_user_data_creation(self):
 		user_data = json.loads(get_user_data('test_privacy@example.com'))
@@ -45,7 +45,7 @@ class TestRequestPersonalData(unittest.TestCase):
 			limit=1)
 		self.assertTrue("Subject: Download Your Data" in email_queue[0].message)
 
-		frappe.db.sql("delete from `tabEmail Queue`")
+		frappe.db.delete("Email Queue")
 
 def create_user_if_not_exists(email, first_name = None):
 	frappe.delete_doc_if_exists("User", email)

frappe/website/doctype/web_page/test_web_page.py

@@ -8,7 +8,7 @@ test_records = frappe.get_test_records('Web Page')
 
 class TestWebPage(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql("delete from `tabWeb Page`")
+		frappe.db.delete("Web Page")
 		for t in test_records:
 			frappe.get_doc(t).insert()
 

frappe/website/doctype/website_settings/website_settings.json

@@ -56,9 +56,11 @@
   "google_analytics_id",
   "google_analytics_anonymize_ip",
   "misc_section",
-  "subdomain",
+  "app_name",
+  "app_logo",
   "disable_signup",
   "section_break_38",
+  "subdomain",
   "head_html",
   "robots_txt",
   "route_redirects",
@@ -224,7 +226,7 @@
    "collapsible": 1,
    "fieldname": "misc_section",
    "fieldtype": "Section Break",
-   "label": "Disable Signup"
+   "label": "Login Page"
   },
   {
    "description": "An icon file with .ico extension. Should be 16 x 16 px. Generated using a favicon generator. [favicon-generator.org]",
@@ -235,7 +237,7 @@
   {
    "description": "Sub-domain provided by erpnext.com",
    "fieldname": "subdomain",
-   "fieldtype": "Text",
+   "fieldtype": "Small Text",
    "label": "Subdomain",
    "read_only": 1
   },
@@ -425,6 +427,17 @@
    "fieldname": "navbar_template_section",
    "fieldtype": "Section Break",
    "label": "Navbar Template"
+  },
+  {
+   "default": "Frappe",
+   "fieldname": "app_name",
+   "fieldtype": "Data",
+   "label": "App Name"
+  },
+  {
+   "fieldname": "app_logo",
+   "fieldtype": "Attach Image",
+   "label": "App Logo"
   }
  ],
  "icon": "fa fa-cog",
@@ -433,7 +446,7 @@
  "issingle": 1,
  "links": [],
  "max_attachments": 10,
- "modified": "2021-07-15 17:39:56.609771",
+ "modified": "2021-08-23 21:39:51.702248",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Website Settings",
@@ -457,4 +470,4 @@
  "sort_field": "modified",
  "sort_order": "ASC",
  "track_changes": 1
-}
+}

frappe/workflow/doctype/workflow/test_workflow.py

@@ -76,7 +76,7 @@ class TestWorkflow(unittest.TestCase):
 		self.assertListEqual(actions, ['Review'])
 
 	def test_if_workflow_actions_were_processed(self):
-		frappe.db.sql('delete from `tabWorkflow Action`')
+		frappe.db.delete("Workflow Action")
 		user = frappe.get_doc('User', 'test2@example.com')
 		user.add_roles('Test Approver', 'System Manager')
 		frappe.set_user('test2@example.com')

frappe/www/login.py

@@ -34,9 +34,11 @@ def get_context(context):
 	context.for_test = 'login.html'
 	context["title"] = "Login"
 	context["provider_logins"] = []
-	context["disable_signup"] = frappe.utils.cint(frappe.db.get_value("Website Settings", "Website Settings", "disable_signup"))
-	context["logo"] = frappe.get_hooks("app_logo_url")[-1]
-	context["app_name"] = frappe.get_system_settings("app_name") or _("Frappe")
+	context["disable_signup"] = frappe.utils.cint(frappe.db.get_single_value("Website Settings", "disable_signup"))
+	context["logo"] = (frappe.db.get_single_value('Website Settings', 'app_logo') or
+		frappe.get_hooks("app_logo_url")[-1])
+	context["app_name"] = (frappe.db.get_single_value('Website Settings', 'app_name') or
+		frappe.get_system_settings("app_name") or _("Frappe"))
 	providers = [i.name for i in frappe.get_all("Social Login Key", filters={"enable_social_login":1}, order_by="name")]
 	for provider in providers:
 		client_id, base_url = frappe.get_value("Social Login Key", provider, ["client_id", "base_url"])

package.json

@@ -35,7 +35,7 @@
     "express": "^4.17.1",
     "fast-deep-equal": "^2.0.1",
     "frappe-charts": "^2.0.0-rc13",
-    "frappe-datatable": "^1.15.3",
+    "frappe-datatable": "^1.15.4",
     "frappe-gantt": "^0.5.0",
     "fuse.js": "^3.4.6",
     "highlight.js": "^10.4.1",

yarn.lock

@@ -2721,10 +2721,10 @@ frappe-charts@^2.0.0-rc13:
   resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc13.tgz#fdb251d7ae311c41e38f90a3ae108070ec6b9072"
   integrity sha512-Bv7IfllIrjRbKWHn5b769dOSenqdBixAr6m5kurf8ZUOJSLOgK4HOXItJ7BA8n9PvviH9/k5DaloisjLM2Bm1w==
 
-frappe-datatable@^1.15.3:
-  version "1.15.3"
-  resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.15.3.tgz#1737e9aebfd363ffadffced71a3534c40e350223"
-  integrity sha512-tUE3pNbxCMX0HPKvwurLBPRAOAdS0gNo1+MpoyFSqXI7b7sp6/TCBRht6qu1Luw+VyIzBtXkJdnnqU+Uoy8iow==
+frappe-datatable@^1.15.4:
+  version "1.15.4"
+  resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.15.4.tgz#dc2e5e5d8a0a7cb8ee658f2d39966af1d4405401"
+  integrity sha512-eW3upPvverm1GNBL4+IcPDvjm5xbJc5ZXW8TYEUZt/QQ2W75K/T6736pSzi9D6mX9sn3BtZ7Ige7MS45SGrgzQ==
   dependencies:
     hyperlist "^1.0.0-beta"
     lodash "^4.17.5"