Merge branch 'develop' into google_drive_picker

.github/workflows/patch-mariadb-tests.yml

@@ -0,0 +1,83 @@
+name: Patch
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    name: Patch Test
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Add to Hosts
+        run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: server
+
+      - name: Run Patch Tests
+        run: |
+          cd ~/frappe-bench/
+          wget https://frappeframework.com/files/v10-frappe.sql.gz
+          bench --site test_site --force restore ~/frappe-bench/v10-frappe.sql.gz
+          bench --site test_site migrate

.github/workflows/server-mariadb-tests.yml

@@ -91,7 +91,6 @@ jobs:
           DB: mariadb
           TYPE: server
 
-
       - name: Run Tests
         run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:

cypress/integration/form.js

@@ -18,6 +18,7 @@ context('Form', () => {
 		cy.get('.primary-action').click();
 		cy.wait('@form_save').its('response.statusCode').should('eq', 200);
 		cy.visit('/app/todo');
+		cy.wait(300);
 		cy.get('.title-text').should('be.visible').and('contain', 'To Do');
 		cy.get('.list-row').should('contain', 'this is a test todo');
 	});

esbuild/esbuild.js

@@ -258,12 +258,17 @@ function get_watch_config() {
 async function clean_dist_folders(apps) {
 	for (let app of apps) {
 		let public_path = get_public_path(app);
-		await fs.promises.rmdir(path.resolve(public_path, "dist", "js"), {
-			recursive: true
-		});
-		await fs.promises.rmdir(path.resolve(public_path, "dist", "css"), {
-			recursive: true
-		});
+		let paths = [
+			path.resolve(public_path, "dist", "js"),
+			path.resolve(public_path, "dist", "css")
+		];
+		for (let target of paths) {
+			if (fs.existsSync(target)) {
+				// rmdir is deprecated in node 16, this will work in both node 14 and 16
+				let rmdir = fs.promises.rm || fs.promises.rmdir;
+				await rmdir(target, { recursive: true });
+			}
+		}
 	}
 }
 

frappe/__init__.py

@@ -1110,9 +1110,7 @@ def setup_module_map():
 
 	if not (local.app_modules and local.module_app):
 		local.module_app, local.app_modules = {}, {}
-		for app in get_all_apps(True):
-			if app == "webnotes":
-				app = "frappe"
+		for app in get_all_apps(with_internal_apps=True):
 			local.app_modules.setdefault(app, [])
 			for module in get_module_list(app):
 				module = scrub(module)
@@ -1493,7 +1491,7 @@ def get_print(doctype=None, name=None, print_format=None, style=None,
 	:param style: Print Format style.
 	:param as_pdf: Return as PDF. Default False.
 	:param password: Password to encrypt the pdf with. Default None"""
-	from frappe.website.render import build_page
+	from frappe.website.serve import get_response_content
 	from frappe.utils.pdf import get_pdf
 
 	local.form_dict.doctype = doctype
@@ -1508,7 +1506,7 @@ def get_print(doctype=None, name=None, print_format=None, style=None,
 		options = {'password': password}
 
 	if not html:
-		html = build_page("printview")
+		html = get_response_content("printview")
 
 	if as_pdf:
 		return get_pdf(html, output = output, options = options)

frappe/app.py

@@ -16,9 +16,9 @@ import frappe.handler
 import frappe.auth
 import frappe.api
 import frappe.utils.response
-import frappe.website.render
 from frappe.utils import get_site_name, sanitize_html
 from frappe.middlewares import StaticDataMiddleware
+from frappe.website.serve import get_response
 from frappe.utils.error import make_error_snapshot
 from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request
 from frappe import _
@@ -72,7 +72,7 @@ def application(request):
 			response = frappe.utils.response.download_private_file(request.path)
 
 		elif request.method in ('GET', 'HEAD', 'POST'):
-			response = frappe.website.render.render()
+			response = get_response()
 
 		else:
 			raise NotFound
@@ -266,8 +266,7 @@ def handle_exception(e):
 		make_error_snapshot(e)
 
 	if return_as_message:
-		response = frappe.website.render.render("message",
-			http_status_code=http_status_code)
+		response = get_response("message", http_status_code=http_status_code)
 
 	return response
 

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -333,7 +333,7 @@ class AutoRepeat(Document):
 		if self.reference_doctype and self.reference_document:
 			res = get_contacts_linking_to(self.reference_doctype, self.reference_document, fields=['email_id'])
 			res += get_contacts_linked_from(self.reference_doctype, self.reference_document, fields=['email_id'])
-			email_ids = list(set([d.email_id for d in res]))
+			email_ids = {d.email_id for d in res}
 			if not email_ids:
 				frappe.msgprint(_('No contacts linked to document'), alert=True)
 			else:

frappe/cache_manager.py

@@ -53,7 +53,7 @@ def clear_domain_cache(user=None):
 	cache.delete_value(domain_cache_keys)
 
 def clear_global_cache():
-	from frappe.website.render import clear_cache as clear_website_cache
+	from frappe.website.utils import clear_website_cache
 
 	clear_doctype_cache()
 	clear_website_cache()

frappe/commands/utils.py

@@ -69,14 +69,14 @@ def watch(apps=None):
 def clear_cache(context):
 	"Clear cache, doctype cache and defaults"
 	import frappe.sessions
-	import frappe.website.render
+	from frappe.website.utils import clear_website_cache
 	from frappe.desk.notifications import clear_notifications
 	for site in context.sites:
 		try:
 			frappe.connect(site)
 			frappe.clear_cache()
 			clear_notifications()
-			frappe.website.render.clear_cache()
+			clear_website_cache()
 		finally:
 			frappe.destroy()
 	if not context.sites:
@@ -86,12 +86,12 @@ def clear_cache(context):
 @pass_context
 def clear_website_cache(context):
 	"Clear website cache"
-	import frappe.website.render
+	from frappe.website.utils import clear_website_cache
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			frappe.website.render.clear_cache()
+			clear_website_cache()
 		finally:
 			frappe.destroy()
 	if not context.sites:
@@ -572,22 +572,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 
 		# Generate coverage report only for app that is being tested
 		source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe')
-		omit=[
-			'*.html',
+		incl = [
+			'*.py',
+		]
+		omit = [
 			'*.js',
 			'*.xml',
+			'*.pyc',
 			'*.css',
 			'*.less',
 			'*.scss',
 			'*.vue',
+			'*.html',
+			'*/test_*',
+			'*/node_modules/*',
 			'*/doctype/*/*_dashboard.py',
-			'*/patches/*'
+			'*/patches/*',
 		]
 
 		if not app or app == 'frappe':
+			omit.append('*/tests/*')
 			omit.append('*/commands/*')
 
-		cov = Coverage(source=[source_path], omit=omit)
+		cov = Coverage(source=[source_path], omit=omit, include=incl)
 		cov.start()
 
 	ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,

frappe/contacts/address_and_contact.py

@@ -153,7 +153,7 @@ def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, fil
 	doctypes = frappe.db.get_all("DocField", filters=filters, fields=["parent"],
 		distinct=True, as_list=True)
 
-	doctypes = tuple([d for d in doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE)])
+	doctypes = tuple(d for d in doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE))
 
 	filters.update({
 		"dt": ("not in", [d[0] for d in doctypes])

frappe/contacts/doctype/address/address.py

@@ -257,7 +257,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters):
 
 def get_condensed_address(doc):
 	fields = ["address_title", "address_line1", "address_line2", "city", "county", "state", "country"]
-	return ", ".join([doc.get(d) for d in fields if doc.get(d)])
+	return ", ".join(doc.get(d) for d in fields if doc.get(d))
 
 def update_preferred_address(address, field):
 	frappe.db.set_value('Address', address, field, 0)

frappe/core/doctype/comment/comment.py

@@ -9,7 +9,7 @@ from frappe.core.doctype.user.user import extract_mentions
 from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\
 	get_title, get_title_html
 from frappe.utils import get_fullname
-from frappe.website.render import clear_cache
+from frappe.website.utils import clear_cache
 from frappe.database.schema import add_column
 from frappe.exceptions import ImplicitCommitError
 

frappe/core/doctype/communication/communication.py

@@ -6,10 +6,11 @@ import frappe
 from frappe import _
 from frappe.model.document import Document
 from frappe.utils import validate_email_address, strip_html, cstr, time_diff_in_seconds
-from frappe.core.doctype.communication.email import validate_email, notify, _notify
+from frappe.core.doctype.communication.email import validate_email
+from frappe.core.doctype.communication.mixins import CommunicationEmailMixin
 from frappe.core.utils import get_parent_doc
 from frappe.utils.bot import BotReply
-from frappe.utils import parse_addr
+from frappe.utils import parse_addr, split_emails
 from frappe.core.doctype.comment.comment import update_comment_in_doc
 from email.utils import parseaddr
 from urllib.parse import unquote
@@ -19,7 +20,7 @@ from frappe.automation.doctype.assignment_rule.assignment_rule import apply as a
 
 exclude_from_linked_with = True
 
-class Communication(Document):
+class Communication(Document, CommunicationEmailMixin):
 	"""Communication represents an external communication like Email.
 	"""
 	no_feed_on_delete = True
@@ -125,6 +126,45 @@ class Communication(Document):
 		if self.communication_type == "Communication":
 			self.notify_change('delete')
 
+	@property
+	def sender_mailid(self):
+		return parse_addr(self.sender)[1] if self.sender else ""
+
+	@staticmethod
+	def _get_emails_list(emails=None, exclude_displayname = False):
+		"""Returns list of emails from given email string.
+
+		* Removes duplicate mailids
+		* Removes display name from email address if exclude_displayname is True
+		"""
+		emails = split_emails(emails) if isinstance(emails, str) else (emails or [])
+		if exclude_displayname:
+			return [email.lower() for email in set([parse_addr(email)[1] for email in emails]) if email]
+		return [email.lower() for email in set(emails) if email]
+
+	def to_list(self, exclude_displayname = True):
+		"""Returns to list.
+		"""
+		return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname)
+
+	def cc_list(self, exclude_displayname = True):
+		"""Returns cc list.
+		"""
+		return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname)
+
+	def bcc_list(self, exclude_displayname = True):
+		"""Returns bcc list.
+		"""
+		return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname)
+
+	def get_attachments(self):
+		attachments = frappe.get_all(
+			"File",
+			fields=["name", "file_name", "file_url", "is_private"],
+			filters = {"attached_to_name": self.name, "attached_to_doctype": self.DOCTYPE}
+		)
+		return attachments
+
 	def notify_change(self, action):
 		frappe.publish_realtime('update_docinfo_for_{}_{}'.format(self.reference_doctype, self.reference_name), {
 			'doc': self.as_dict(),
@@ -198,36 +238,6 @@ class Communication(Document):
 				if not self.sender_full_name:
 					self.sender_full_name = sender_email
 
-	def send(self, print_html=None, print_format=None, attachments=None,
-		send_me_a_copy=False, recipients=None):
-		"""Send communication via Email.
-
-		:param print_html: Send given value as HTML attachment.
-		:param print_format: Attach print format of parent document."""
-
-		self.send_me_a_copy = send_me_a_copy
-		self.notify(print_html, print_format, attachments, recipients)
-
-	def notify(self, print_html=None, print_format=None, attachments=None,
-		recipients=None, cc=None, bcc=None,fetched_from_email_account=False):
-		"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
-
-		:param print_html: Send given value as HTML attachment
-		:param print_format: Attach print format of parent document
-		:param attachments: A list of filenames that should be attached when sending this email
-		:param recipients: Email recipients
-		:param cc: Send email as CC to
-		:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
-
-		"""
-		notify(self, print_html, print_format, attachments, recipients, cc, bcc,
-			fetched_from_email_account)
-
-	def _notify(self, print_html=None, print_format=None, attachments=None,
-		recipients=None, cc=None, bcc=None):
-
-		_notify(self, print_html, print_format, attachments, recipients, cc, bcc)
-
 	def bot_reply(self):
 		if self.comment_type == 'Bot' and self.communication_type == 'Chat':
 			reply = BotReply().get_reply(self.content)
@@ -504,3 +514,4 @@ def set_avg_response_time(parent, communication):
 			if response_times:
 				avg_response_time = sum(response_times) / len(response_times)
 				parent.db_set("avg_response_time", avg_response_time)
+

frappe/core/doctype/communication/email.py

@@ -13,6 +13,11 @@ import time
 from frappe import _
 from frappe.utils.background_jobs import enqueue
 
+OUTGOING_EMAIL_ACCOUNT_MISSING = _("""
+	Unable to send mail because of a missing email account.
+	Please setup default Email Account from Setup > Email > Email Account
+""")
+
 @frappe.whitelist()
 def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
 	sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False,
@@ -36,7 +41,6 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 	:param send_me_a_copy: Send a copy to the sender (default **False**).
 	:param email_template: Template which is used to compose mail .
 	"""
-
 	is_error_report = (doctype=="User" and name==frappe.session.user and subject=="Error Report")
 	send_me_a_copy = cint(send_me_a_copy)
 
@@ -84,12 +88,16 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 	frappe.db.commit()
 
 	if cint(send_email):
-		frappe.flags.print_letterhead = cint(print_letterhead)
-		comm.send(print_html, print_format, attachments, send_me_a_copy=send_me_a_copy)
+		if not comm.get_outgoing_email_account():
+			frappe.throw(msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError)
 
+		comm.send_email(print_html=print_html, print_format=print_format,
+			send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead)
+
+	emails_not_sent_to = comm.exclude_emails_list(include_sender=send_me_a_copy)
 	return {
 		"name": comm.name,
-		"emails_not_sent_to": ", ".join(comm.emails_not_sent_to) if hasattr(comm, "emails_not_sent_to") else None
+		"emails_not_sent_to": ", ".join(emails_not_sent_to or [])
 	}
 
 def validate_email(doc):
@@ -110,164 +118,6 @@ def validate_email(doc):
 
 	# validate sender
 
-def notify(doc, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, bcc=None, fetched_from_email_account=False):
-	"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
-
-	:param print_html: Send given value as HTML attachment
-	:param print_format: Attach print format of parent document
-	:param attachments: A list of filenames that should be attached when sending this email
-	:param recipients: Email recipients
-	:param cc: Send email as CC to
-	:param bcc: Send email as BCC to
-	:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
-
-	"""
-	recipients, cc, bcc = get_recipients_cc_and_bcc(doc, recipients, cc, bcc,
-		fetched_from_email_account=fetched_from_email_account)
-
-	if not recipients and not cc:
-		return
-
-	doc.emails_not_sent_to = set(doc.all_email_addresses) - set(doc.sent_email_addresses)
-
-	if frappe.flags.in_test:
-		# for test cases, run synchronously
-		doc._notify(print_html=print_html, print_format=print_format, attachments=attachments,
-			recipients=recipients, cc=cc, bcc=None)
-	else:
-		enqueue(sendmail, queue="default", timeout=300, event="sendmail",
-			communication_name=doc.name,
-			print_html=print_html, print_format=print_format, attachments=attachments,
-			recipients=recipients, cc=cc, bcc=bcc, lang=frappe.local.lang,
-			session=frappe.local.session, print_letterhead=frappe.flags.print_letterhead)
-
-def _notify(doc, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, bcc=None):
-
-	prepare_to_notify(doc, print_html, print_format, attachments)
-
-	if doc.outgoing_email_account.send_unsubscribe_message:
-		unsubscribe_message = _("Leave this conversation")
-	else:
-		unsubscribe_message = ""
-
-	frappe.sendmail(
-		recipients=(recipients or []),
-		cc=(cc or []),
-		bcc=(bcc or []),
-		expose_recipients="header",
-		sender=doc.sender,
-		reply_to=doc.incoming_email_account,
-		subject=doc.subject,
-		content=doc.content,
-		reference_doctype=doc.reference_doctype,
-		reference_name=doc.reference_name,
-		attachments=doc.attachments,
-		message_id=doc.message_id,
-		unsubscribe_message=unsubscribe_message,
-		delayed=True,
-		communication=doc.name,
-		read_receipt=doc.read_receipt,
-		is_notification=True if doc.sent_or_received =="Received" else False,
-		print_letterhead=frappe.flags.print_letterhead
-	)
-
-def get_recipients_cc_and_bcc(doc, recipients, cc, bcc, fetched_from_email_account=False):
-	doc.all_email_addresses = []
-	doc.sent_email_addresses = []
-	doc.previous_email_sender = None
-
-	if not recipients:
-		recipients = get_recipients(doc, fetched_from_email_account=fetched_from_email_account)
-
-	if not cc:
-		cc = get_cc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
-
-	if not bcc:
-		bcc = get_bcc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
-
-	if fetched_from_email_account:
-		# email was already sent to the original recipient by the sender's email service
-		original_recipients, recipients = recipients, []
-
-		# send email to the sender of the previous email in the thread which this email is a reply to
-		#provides erratic results and can send external
-		#if doc.previous_email_sender:
-		#	recipients.append(doc.previous_email_sender)
-
-		# cc that was received in the email
-		original_cc = split_emails(doc.cc)
-
-		# don't cc to people who already received the mail from sender's email service
-		cc = list(set(cc) - set(original_cc) - set(original_recipients))
-		remove_administrator_from_email_list(cc)
-
-		original_bcc = split_emails(doc.bcc)
-		bcc = list(set(bcc) - set(original_bcc) - set(original_recipients))
-		remove_administrator_from_email_list(bcc)
-
-	remove_administrator_from_email_list(recipients)
-
-	return recipients, cc, bcc
-
-def remove_administrator_from_email_list(email_list):
-	administrator_email = list(filter(lambda emails: "Administrator" in emails, email_list))
-	if administrator_email:
-		email_list.remove(administrator_email[0])
-
-def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None):
-	"""Prepare to make multipart MIME Email
-
-	:param print_html: Send given value as HTML attachment.
-	:param print_format: Attach print format of parent document."""
-
-	view_link = frappe.utils.cint(frappe.db.get_value("System Settings", "System Settings", "attach_view_link"))
-
-	if print_format and view_link:
-		doc.content += get_attach_link(doc, print_format)
-
-	set_incoming_outgoing_accounts(doc)
-
-	if not doc.sender:
-		doc.sender = doc.outgoing_email_account.email_id
-
-	if not doc.sender_full_name:
-		doc.sender_full_name = doc.outgoing_email_account.name or _("Notification")
-
-	if doc.sender:
-		# combine for sending to get the format 'Jane <jane@example.com>'
-		doc.sender = get_formatted_email(doc.sender_full_name, mail=doc.sender)
-
-	doc.attachments = []
-
-	if print_html or print_format:
-		doc.attachments.append({"print_format_attachment":1, "doctype":doc.reference_doctype,
-			"name":doc.reference_name, "print_format":print_format, "html":print_html})
-
-	if attachments:
-		if isinstance(attachments, str):
-			attachments = json.loads(attachments)
-
-		for a in attachments:
-			if isinstance(a, str):
-				# is it a filename?
-				try:
-					# check for both filename and file id
-					file_id = frappe.db.get_list('File', or_filters={'file_name': a, 'name': a}, limit=1)
-					if not file_id:
-						frappe.throw(_("Unable to find attachment {0}").format(a))
-					file_id = file_id[0]['name']
-					_file = frappe.get_doc("File", file_id)
-					_file.get_content()
-					# these attachments will be attached on-demand
-					# and won't be stored in the message
-					doc.attachments.append({"fid": file_id})
-				except IOError:
-					frappe.throw(_("Unable to find attachment {0}").format(a))
-			else:
-				doc.attachments.append(a)
-
 def set_incoming_outgoing_accounts(doc):
 	from frappe.email.doctype.email_account.email_account import EmailAccount
 	incoming_email_account = EmailAccount.find_incoming(
@@ -280,74 +130,6 @@ def set_incoming_outgoing_accounts(doc):
 	if doc.sent_or_received == "Sent":
 		doc.db_set("email_account", doc.outgoing_email_account.name)
 
-def get_recipients(doc, fetched_from_email_account=False):
-	"""Build a list of email addresses for To"""
-	# [EDGE CASE] doc.recipients can be None when an email is sent as BCC
-	recipients = split_emails(doc.recipients)
-
-	#if fetched_from_email_account and doc.in_reply_to:
-		# add sender of previous reply
-		#doc.previous_email_sender = frappe.db.get_value("Communication", doc.in_reply_to, "sender")
-		#recipients.append(doc.previous_email_sender)
-
-	if recipients:
-		recipients = filter_email_list(doc, recipients, [])
-
-	return recipients
-
-def get_cc(doc, recipients=None, fetched_from_email_account=False):
-	"""Build a list of email addresses for CC"""
-	# get a copy of CC list
-	cc = split_emails(doc.cc)
-
-	if doc.reference_doctype and doc.reference_name:
-		if fetched_from_email_account:
-			# if it is a fetched email, add follows to CC
-			cc.append(get_owner_email(doc))
-			cc += get_assignees(doc)
-
-	if getattr(doc, "send_me_a_copy", False) and doc.sender not in cc:
-		cc.append(doc.sender)
-
-	if cc:
-		# exclude unfollows, recipients and unsubscribes
-		exclude = [] #added to remove account check
-		exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)]
-		exclude += [(parse_addr(email)[1] or "").lower() for email in recipients]
-
-		if fetched_from_email_account:
-			# exclude sender when pulling email
-			exclude += [parse_addr(doc.sender)[1]]
-
-		if doc.reference_doctype and doc.reference_name:
-			exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"],
-				{"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)]
-
-		cc = filter_email_list(doc, cc, exclude, is_cc=True)
-
-	return cc
-
-def get_bcc(doc, recipients=None, fetched_from_email_account=False):
-	"""Build a list of email addresses for BCC"""
-	bcc = split_emails(doc.bcc)
-
-	if bcc:
-		exclude = []
-		exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)]
-		exclude += [(parse_addr(email)[1] or "").lower() for email in recipients]
-
-		if fetched_from_email_account:
-			# exclude sender when pulling email
-			exclude += [parse_addr(doc.sender)[1]]
-
-		if doc.reference_doctype and doc.reference_name:
-			exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"],
-				{"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)]
-
-		bcc = filter_email_list(doc, bcc, exclude, is_bcc=True)
-
-	return bcc
-
 def add_attachments(name, attachments):
 	'''Add attachments to the given Communication'''
 	# loop through attachments
@@ -355,7 +137,6 @@ def add_attachments(name, attachments):
 		if isinstance(a, str):
 			attach = frappe.db.get_value("File", {"name":a},
 				["file_name", "file_url", "is_private"], as_dict=1)
-
 			# save attachments to new doc
 			_file = frappe.get_doc({
 				"doctype": "File",
@@ -367,103 +148,6 @@ def add_attachments(name, attachments):
 			})
 			_file.save(ignore_permissions=True)
 
-def filter_email_list(doc, email_list, exclude, is_cc=False, is_bcc=False):
-	# temp variables
-	filtered = []
-	email_address_list = []
-
-	for email in list(set(email_list)):
-		email_address = (parse_addr(email)[1] or "").lower()
-		if not email_address:
-			continue
-
-		# this will be used to eventually find email addresses that aren't sent to
-		doc.all_email_addresses.append(email_address)
-
-		if (email in exclude) or (email_address in exclude):
-			continue
-
-		if is_cc:
-			is_user_enabled = frappe.db.get_value("User", email_address, "enabled")
-			if is_user_enabled==0:
-				# don't send to disabled users
-				continue
-
-		if is_bcc:
-			is_user_enabled = frappe.db.get_value("User", email_address, "enabled")
-			if is_user_enabled==0:
-				continue
-
-		# make sure of case-insensitive uniqueness of email address
-		if email_address not in email_address_list:
-			# append the full email i.e. "Human <human@example.com>"
-			filtered.append(email)
-			email_address_list.append(email_address)
-
-	doc.sent_email_addresses.extend(email_address_list)
-
-	return filtered
-
-def get_owner_email(doc):
-	owner = get_parent_doc(doc).owner
-	return get_formatted_email(owner) or owner
-
-def get_assignees(doc):
-	return [( get_formatted_email(d.owner) or d.owner ) for d in
-		frappe.db.get_all("ToDo", filters={
-			"reference_type": doc.reference_doctype,
-			"reference_name": doc.reference_name,
-			"status": "Open"
-		}, fields=["owner"])
-	]
-
-def get_attach_link(doc, print_format):
-	"""Returns public link for the attachment via `templates/emails/print_link.html`."""
-	return frappe.get_template("templates/emails/print_link.html").render({
-		"url": get_url(),
-		"doctype": doc.reference_doctype,
-		"name": doc.reference_name,
-		"print_format": print_format,
-		"key": get_parent_doc(doc).get_signature()
-	})
-
-def sendmail(communication_name, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, bcc=None, lang=None, session=None, print_letterhead=None):
-	try:
-
-		if lang:
-			frappe.local.lang = lang
-
-		if session:
-			# hack to enable access to private files in PDF
-			session['data'] = frappe._dict(session['data'])
-			frappe.local.session.update(session)
-
-		if print_letterhead:
-			frappe.flags.print_letterhead = print_letterhead
-
-		# upto 3 retries
-		for i in range(3):
-			try:
-				communication = frappe.get_doc("Communication", communication_name)
-				communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
-					recipients=recipients, cc=cc, bcc=bcc)
-
-			except frappe.db.InternalError as e:
-				# deadlock, try again
-				if frappe.db.is_deadlocked(e):
-					frappe.db.rollback()
-					time.sleep(1)
-					continue
-				else:
-					raise
-			else:
-				break
-
-	except:
-		traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail")
-		raise
-
 @frappe.whitelist(allow_guest=True)
 def mark_email_as_seen(name=None):
 	try:

frappe/core/doctype/communication/mixins.py

@@ -0,0 +1,297 @@
+import frappe
+from frappe import _
+from frappe.core.utils import get_parent_doc
+from frappe.utils import parse_addr, get_formatted_email, get_url
+from frappe.email.doctype.email_account.email_account import EmailAccount
+
+class CommunicationEmailMixin:
+	"""Mixin class to handle communication mails.
+	"""
+	def is_email_communication(self):
+		return self.communication_type=="Communication" and self.communication_medium == "Email"
+
+	def get_owner(self):
+		"""Get owner of the communication docs parent.
+		"""
+		parent_doc = get_parent_doc(self)
+		return parent_doc.owner if parent_doc else None
+
+	def get_all_email_addresses(self, exclude_displayname=False):
+		"""Get all Email addresses mentioned in the doc along with display name.
+		"""
+		return self.to_list(exclude_displayname=exclude_displayname) + \
+			self.cc_list(exclude_displayname=exclude_displayname) + \
+			self.bcc_list(exclude_displayname=exclude_displayname)
+
+	def get_email_with_displayname(self, email_address):
+		"""Returns email address after adding displayname.
+		"""
+		display_name, email = parse_addr(email_address)
+		if display_name and display_name != email:
+			return email_address
+
+		# emailid to emailid with display name map.
+		email_map = {parse_addr(email)[1]: email for email in self.get_all_email_addresses()}
+		return email_map.get(email, email)
+
+	def mail_recipients(self, is_inbound_mail_communcation=False):
+		"""Build to(recipient) list to send an email.
+		"""
+		# Incase of inbound mail, recipients already received the mail, no need to send again.
+		if is_inbound_mail_communcation:
+			return []
+
+		if hasattr(self, '_final_recipients'):
+			return self._final_recipients
+
+		to = self.to_list()
+		self._final_recipients = list(filter(lambda id: id != 'Administrator', to))
+		return self._final_recipients
+
+	def get_mail_recipients_with_displayname(self, is_inbound_mail_communcation=False):
+		"""Build to(recipient) list to send an email including displayname in email.
+		"""
+		to_list = self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)
+		return [self.get_email_with_displayname(email) for email in to_list]
+
+	def mail_cc(self, is_inbound_mail_communcation=False, include_sender = False):
+		"""Build cc list to send an email.
+
+		* if email copy is requested by sender, then add sender to CC.
+		* If this doc is created through inbound mail, then add doc owner to cc list
+		* remove all the thread_notify disabled users.
+		* Make sure that all users enabled in the system
+		* Remove admin from email list
+
+		* FixMe: Removed adding TODO owners to cc list. Check if that is needed.
+		"""
+		if hasattr(self, '_final_cc'):
+			return self._final_cc
+
+		cc = self.cc_list()
+
+		# Need to inform parent document owner incase communication is created through inbound mail
+		if include_sender:
+			cc.append(self.sender_mailid)
+		if is_inbound_mail_communcation:
+			cc.append(self.get_owner())
+			cc = set(cc) - {self.sender_mailid}
+
+		cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc))
+		cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
+		cc = cc - set(self.filter_disabled_users(cc))
+
+		# # Incase of inbound mail, to and cc already received the mail, no need to send again.
+		if is_inbound_mail_communcation:
+			cc = cc - set(self.cc_list() + self.to_list())
+
+		self._final_cc = list(filter(lambda id: id != 'Administrator', cc))
+		return self._final_cc
+
+	def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender = False):
+		cc_list = self.mail_cc(is_inbound_mail_communcation=False, include_sender = False)
+		return [self.get_email_with_displayname(email) for email in cc_list]
+
+	def mail_bcc(self, is_inbound_mail_communcation=False):
+		"""
+		* Thread_notify check
+		* Email unsubscribe list
+		* User must be enabled in the system
+		* remove_administrator_from_email_list
+		"""
+		if hasattr(self, '_final_bcc'):
+			return self._final_bcc
+
+		bcc = set(self.bcc_list())
+		if is_inbound_mail_communcation:
+			bcc = bcc - {self.sender_mailid}
+		bcc = bcc - set(self.filter_thread_notification_disbled_users(bcc))
+		bcc = bcc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
+		bcc = bcc - set(self.filter_disabled_users(bcc))
+
+		# Incase of inbound mail, to and cc & bcc already received the mail, no need to send again.
+		if is_inbound_mail_communcation:
+			bcc = bcc - set(self.bcc_list() + self.to_list())
+
+		self._final_bcc = list(filter(lambda id: id != 'Administrator', bcc))
+		return self._final_bcc
+
+	def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False):
+		bcc_list = self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation)
+		return [self.get_email_with_displayname(email) for email in bcc_list]
+
+	def mail_sender(self):
+		email_account = self.get_outgoing_email_account()
+		if not self.sender_mailid and email_account:
+			return email_account.email_id
+		return self.sender_mailid
+
+	def mail_sender_fullname(self):
+		email_account = self.get_outgoing_email_account()
+		if not self.sender_full_name:
+			return (email_account and email_account.name) or _("Notification")
+		return self.sender_full_name
+
+	def get_mail_sender_with_displayname(self):
+		return get_formatted_email(self.mail_sender_fullname(), mail=self.mail_sender())
+
+	def get_content(self, print_format=None):
+		if print_format:
+			return self.content + self.get_attach_link(print_format)
+		return self.content
+
+	def get_attach_link(self, print_format):
+		"""Returns public link for the attachment via `templates/emails/print_link.html`."""
+		return frappe.get_template("templates/emails/print_link.html").render({
+			"url": get_url(),
+			"doctype": self.reference_doctype,
+			"name": self.reference_name,
+			"print_format": print_format,
+			"key": get_parent_doc(self).get_signature()
+		})
+
+	def get_outgoing_email_account(self):
+		if not hasattr(self, '_outgoing_email_account'):
+			if self.email_account:
+				self._outgoing_email_account = EmailAccount.find(self.email_account)
+			else:
+				self._outgoing_email_account = EmailAccount.find_outgoing(
+					match_by_email=self.sender_mailid,
+					match_by_doctype=self.reference_doctype
+				)
+
+				if self.sent_or_received == "Sent" and self._outgoing_email_account:
+					self.db_set("email_account", self._outgoing_email_account.name)
+
+		return self._outgoing_email_account
+
+	def get_incoming_email_account(self):
+		if not hasattr(self, '_incoming_email_account'):
+			self._incoming_email_account = EmailAccount.find_incoming(
+				match_by_email=self.sender_mailid,
+				match_by_doctype=self.reference_doctype
+			)
+		return self._incoming_email_account
+
+	def mail_attachments(self, print_format=None, print_html=None):
+		final_attachments = []
+
+		if print_format and print_html:
+			d = {'print_format': print_format, 'print_html': print_html, 'print_format_attachment': 1,
+					'doctype': self.reference_doctype, 'name': self.reference_name}
+			final_attachments.append(d)
+
+		for a in self.get_attachments() or []:
+			final_attachments.append({"fid": a['name']})
+
+		return final_attachments
+
+	def get_unsubscribe_message(self):
+		email_account = self.get_outgoing_email_account()
+		if email_account and email_account.send_unsubscribe_message:
+			return _("Leave this conversation")
+		return ''
+
+	def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False):
+		"""List of mail id's excluded while sending mail.
+		"""
+		all_ids = self.get_all_email_addresses(exclude_displayname=True)
+		final_ids = self.mail_recipients(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
+			self.mail_bcc(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
+			self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender)
+		return set(all_ids) - set(final_ids)
+
+	@staticmethod
+	def filter_thread_notification_disbled_users(emails):
+		"""Filter users based on notifications for email threads setting is disabled.
+		"""
+		if not emails:
+			return []
+
+		disabled_users = frappe.db.sql_list("""
+			SELECT
+				email
+			FROM
+				`tabUser`
+			where
+				email in %(emails)s
+				and
+				thread_notify=0
+		""", {'emails': tuple(emails)})
+		return disabled_users
+
+	@staticmethod
+	def filter_disabled_users(emails):
+		"""
+		"""
+		if not emails:
+			return []
+
+		disabled_users = frappe.db.sql_list("""
+			SELECT
+				email
+			FROM
+				`tabUser`
+			where
+				email in %(emails)s
+				and
+				enabled=0
+		""", {'emails': tuple(emails)})
+		return disabled_users
+
+	def sendmail_input_dict(self, print_html=None, print_format=None,
+			send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None):
+
+		outgoing_email_account = self.get_outgoing_email_account()
+		if not outgoing_email_account:
+			return {}
+
+		recipients = self.get_mail_recipients_with_displayname(
+			is_inbound_mail_communcation=is_inbound_mail_communcation
+		)
+		cc = self.get_mail_cc_with_displayname(
+			is_inbound_mail_communcation=is_inbound_mail_communcation,
+			include_sender = send_me_a_copy
+		)
+		bcc = self.get_mail_bcc_with_displayname(
+			is_inbound_mail_communcation=is_inbound_mail_communcation
+		)
+
+		if not (recipients or cc):
+			return {}
+
+		final_attachments = self.mail_attachments(print_format=print_format, print_html=print_html)
+		incoming_email_account = self.get_incoming_email_account()
+		return {
+			"recipients": recipients,
+			"cc": cc,
+			"bcc": bcc,
+			"expose_recipients": "header",
+			"sender": self.get_mail_sender_with_displayname(),
+			"reply_to": incoming_email_account and incoming_email_account.email_id,
+			"subject": self.subject,
+			"content": self.get_content(print_format=print_format),
+			"reference_doctype": self.reference_doctype,
+			"reference_name": self.reference_name,
+			"attachments": final_attachments,
+			"message_id": self.message_id,
+			"unsubscribe_message": self.get_unsubscribe_message(),
+			"delayed": True,
+			"communication": self.name,
+			"read_receipt": self.read_receipt,
+			"is_notification": (self.sent_or_received =="Received" and True) or False,
+			"print_letterhead": print_letterhead
+		}
+
+	def send_email(self, print_html=None, print_format=None,
+			send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None):
+		input_dict = self.sendmail_input_dict(
+			print_html=print_html,
+			print_format=print_format,
+			send_me_a_copy=send_me_a_copy,
+			print_letterhead=print_letterhead,
+			is_inbound_mail_communcation=is_inbound_mail_communcation
+		)
+
+		if input_dict:
+			frappe.sendmail(**input_dict)

frappe/core/doctype/communication/test_communication.py

@@ -1,10 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # See license.txt
-import frappe
 import unittest
 from urllib.parse import quote
-test_records = frappe.get_test_records('Communication')
 
+import frappe
+from frappe.email.doctype.email_queue.email_queue import EmailQueue
+
+test_records = frappe.get_test_records('Communication')
 
 class TestCommunication(unittest.TestCase):
 
@@ -199,6 +201,70 @@ class TestCommunication(unittest.TestCase):
 
 		self.assertIn(("Note", note.name), doc_links)
 
+class TestCommunicationEmailMixin(unittest.TestCase):
+	def new_communication(self, recipients=None, cc=None, bcc=None):
+		recipients = ', '.join(recipients or [])
+		cc = ', '.join(cc or [])
+		bcc = ', '.join(bcc or [])
+
+		comm = frappe.get_doc({
+			"doctype": "Communication",
+			"communication_type": "Communication",
+			"communication_medium": "Email",
+			"content": "Test content",
+			"recipients": recipients,
+			"cc": cc,
+			"bcc": bcc
+		}).insert(ignore_permissions=True)
+		return comm
+
+	def new_user(self, email, **user_data):
+		user_data.setdefault('first_name', 'first_name')
+		user = frappe.new_doc('User')
+		user.email = email
+		user.update(user_data)
+		user.insert(ignore_permissions=True, ignore_if_duplicate=True)
+		return user
+
+	def test_recipients(self):
+		to_list = ['to@test.com', 'receiver <to+1@test.com>', 'to@test.com']
+		comm = self.new_communication(recipients = to_list)
+		res = comm.get_mail_recipients_with_displayname()
+		self.assertCountEqual(res, ['to@test.com', 'receiver <to+1@test.com>'])
+		comm.delete()
+
+	def test_cc(self):
+		to_list = ['to@test.com']
+		cc_list = ['cc+1@test.com', 'cc <cc+2@test.com>', 'to@test.com']
+		user = self.new_user(email='cc+1@test.com', thread_notify=0)
+		comm = self.new_communication(recipients=to_list, cc=cc_list)
+		res = comm.get_mail_cc_with_displayname()
+		self.assertCountEqual(res, ['cc <cc+2@test.com>'])
+		user.delete()
+		comm.delete()
+
+	def test_bcc(self):
+		bcc_list = ['bcc+1@test.com', 'cc <bcc+2@test.com>', ]
+		user = self.new_user(email='bcc+2@test.com', enabled=0)
+		comm = self.new_communication(bcc=bcc_list)
+		res = comm.get_mail_bcc_with_displayname()
+		self.assertCountEqual(res, ['bcc+1@test.com'])
+		user.delete()
+		comm.delete()
+
+	def test_sendmail(self):
+		to_list = ['to <to@test.com>']
+		cc_list = ['cc <cc+1@test.com>', 'cc <cc+2@test.com>']
+
+		comm = self.new_communication(recipients=to_list, cc=cc_list)
+		comm.send_email()
+		doc = EmailQueue.find_one_by_filters(communication=comm.name)
+		mail_receivers = [each.recipient for each in doc.recipients]
+		self.assertIsNotNone(doc)
+		self.assertCountEqual(to_list+cc_list, mail_receivers)
+		doc.delete()
+		comm.delete()
+
 def create_email_account():
 	frappe.delete_doc_if_exists("Email Account", "_Test Comm Account 1")
 
@@ -229,4 +295,4 @@ def create_email_account():
 		"enable_automatic_linking": 1
 	}).insert(ignore_permissions=True)
 
-	return email_account
+	return email_account

frappe/core/doctype/data_import/importer.py

@@ -449,7 +449,7 @@ class ImportFile:
 			for row in data_without_first_row:
 				row_values = row.get_values(parent_column_indexes)
 				# if the row is blank, it's a child row doc
-				if all([v in INVALID_VALUES for v in row_values]):
+				if all(v in INVALID_VALUES for v in row_values):
 					rows.append(row)
 					continue
 				# if we encounter a row which has values in parent columns,
@@ -606,7 +606,7 @@ class Row:
 		if df.fieldtype == "Select":
 			select_options = get_select_options(df)
 			if select_options and value not in select_options:
-				options_string = ", ".join([frappe.bold(d) for d in select_options])
+				options_string = ", ".join(frappe.bold(d) for d in select_options)
 				msg = _("Value must be one of {0}").format(options_string)
 				self.warnings.append(
 					{"row": self.row_number, "field": df_as_json(df), "message": msg,}
@@ -902,7 +902,7 @@ class Column:
 
 		if self.df.fieldtype == "Link":
 			# find all values that dont exist
-			values = list(set([cstr(v) for v in self.column_values[1:] if v]))
+			values = list({cstr(v) for v in self.column_values[1:] if v})
 			exists = [
 				d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)})
 			]
@@ -935,11 +935,11 @@ class Column:
 		elif self.df.fieldtype == "Select":
 			options = get_select_options(self.df)
 			if options:
-				values = list(set([cstr(v) for v in self.column_values[1:] if v]))
-				invalid = list(set(values) - set(options))
+				values = {cstr(v) for v in self.column_values[1:] if v}
+				invalid = values - set(options)
 				if invalid:
-					valid_values = ", ".join([frappe.bold(o) for o in options])
-					invalid_values = ", ".join([frappe.bold(i) for i in invalid])
+					valid_values = ", ".join(frappe.bold(o) for o in options)
+					invalid_values = ", ".join(frappe.bold(i) for i in invalid)
 					self.warnings.append(
 						{
 							"col": self.column_number,

frappe/core/doctype/data_import_legacy/importer.py

@@ -177,7 +177,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 						if d.get("name") and d["name"].startswith('"'):
 							d["name"] = d["name"][1:-1]
 
-						if sum([0 if not val else 1 for val in d.values()]):
+						if sum(0 if not val else 1 for val in d.values()):
 							d['doctype'] = dt
 							if dt == doctype:
 								doc.update(d)
@@ -533,6 +533,6 @@ def get_parent_field(doctype, parenttype):
 
 def delete_child_rows(rows, doctype):
 	"""delete child rows for all parents"""
-	for p in list(set([r[1] for r in rows])):
+	for p in list(set(r[1] for r in rows)):
 		if p:
 			frappe.db.sql("""delete from `tab{0}` where parent=%s""".format(doctype), p)

frappe/core/doctype/doctype/doctype.py

@@ -8,7 +8,6 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache
 
 # imports - module imports
 import frappe
-import frappe.website.render
 from frappe import _
 from frappe.utils import now, cint
 from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options
@@ -23,6 +22,7 @@ from frappe.model.docfield import supports_translation
 from frappe.modules.import_file import get_file_path
 from frappe.model.meta import Meta
 from frappe.desk.utils import validate_route_conflict
+from frappe.website.utils import clear_cache
 
 class InvalidFieldNameError(frappe.ValidationError): pass
 class UniqueFieldnameError(frappe.ValidationError): pass
@@ -193,7 +193,7 @@ class DocType(Document):
 
 		self.flags.update_fields_to_fetch_queries = []
 
-		if set(old_fields_to_fetch) != set([df.fieldname for df in new_meta.get_fields_to_fetch()]):
+		if set(old_fields_to_fetch) != set(df.fieldname for df in new_meta.get_fields_to_fetch()):
 			for df in new_meta.get_fields_to_fetch():
 				if df.fieldname not in old_fields_to_fetch:
 					link_fieldname, source_fieldname = df.fetch_from.split('.', 1)
@@ -248,7 +248,7 @@ class DocType(Document):
 				frappe.throw(_('Field "route" is mandatory for Web Views'), title='Missing Field')
 
 			# clear website cache
-			frappe.website.render.clear_cache()
+			clear_cache()
 
 	def change_modified_of_parent(self):
 		"""Change the timestamp of parent DocType if the current one is a child to clear caches."""
@@ -550,11 +550,6 @@ class DocType(Document):
 		from frappe.modules.export_file import export_to_files
 		export_to_files(record_list=[['DocType', self.name]], create_init=True)
 
-	def import_doc(self):
-		"""Import from standard folder `[module]/doctype/[name]/[name].json`."""
-		from frappe.modules.import_module import import_from_files
-		import_from_files(record_list=[[self.module, 'doctype', self.name]])
-
 	def make_controller_template(self):
 		"""Make boilerplate controller template."""
 		make_boilerplate("controller._py", self)
@@ -762,7 +757,7 @@ def validate_fields(meta):
 		invalid_fields = ('doctype',)
 		if fieldname in invalid_fields:
 			frappe.throw(_("{0}: Fieldname cannot be one of {1}")
-				.format(docname, ", ".join([frappe.bold(d) for d in invalid_fields])))
+				.format(docname, ", ".join(frappe.bold(d) for d in invalid_fields)))
 
 	def check_unique_fieldname(docname, fieldname):
 		duplicates = list(filter(None, map(lambda df: df.fieldname==fieldname and str(df.idx) or None, fields)))
@@ -996,7 +991,7 @@ def validate_fields(meta):
 			if docfield.options and (docfield.options not in data_field_options):
 				df_str = frappe.bold(_(docfield.label))
 				text_str = _("{0} is an invalid Data field.").format(df_str) + "<br>" * 2 + _("Only Options allowed for Data field are:") + "<br>"
-				df_options_str = "<ul><li>" + "</li><li>".join([_(x) for x in data_field_options]) + "</ul>"
+				df_options_str = "<ul><li>" + "</li><li>".join(_(x) for x in data_field_options) + "</ul>"
 
 				frappe.msgprint(text_str + df_options_str, title="Invalid Data Field", raise_exception=True)
 

frappe/core/doctype/domain/domain.py

@@ -110,7 +110,7 @@ class Domain(Document):
 
 			# enable
 			frappe.db.sql('''update `tabPortal Menu Item` set enabled=1
-				where route in ({0})'''.format(', '.join(['"{0}"'.format(d) for d in self.data.allow_sidebar_items])))
+				where route in ({0})'''.format(', '.join('"{0}"'.format(d) for d in self.data.allow_sidebar_items)))
 
 		if self.data.remove_sidebar_items:
 			# disable all
@@ -118,4 +118,4 @@ class Domain(Document):
 
 			# enable
 			frappe.db.sql('''update `tabPortal Menu Item` set enabled=0
-				where route in ({0})'''.format(', '.join(['"{0}"'.format(d) for d in self.data.remove_sidebar_items])))
+				where route in ({0})'''.format(', '.join('"{0}"'.format(d) for d in self.data.remove_sidebar_items)))

frappe/core/doctype/feedback/feedback.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Feedback', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/feedback/feedback.json

@@ -0,0 +1,86 @@
+{
+ "actions": [],
+ "creation": "2021-06-03 19:02:55.328423",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "reference_doctype",
+  "reference_name",
+  "column_break_3",
+  "email",
+  "rating",
+  "section_break_6",
+  "feedback"
+ ],
+ "fields": [
+  {
+   "fieldname": "column_break_3",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "email",
+   "fieldtype": "Data",
+   "label": "Email",
+   "reqd": 1
+  },
+  {
+   "fieldname": "rating",
+   "fieldtype": "Float",
+   "in_list_view": 1,
+   "label": "Rating",
+   "precision": "1",
+   "reqd": 1
+  },
+  {
+   "fieldname": "section_break_6",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "feedback",
+   "fieldtype": "Small Text",
+   "label": "Feedback",
+   "reqd": 1
+  },
+  {
+   "fieldname": "reference_doctype",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Reference Document Type",
+   "options": "\nBlog Post"
+  },
+  {
+   "fieldname": "reference_name",
+   "fieldtype": "Dynamic Link",
+   "in_list_view": 1,
+   "label": "Reference Name",
+   "options": "reference_doctype",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-06-14 15:11:26.005805",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Feedback",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "title_field": "reference_name",
+ "track_changes": 1
+}

frappe/core/doctype/feedback/feedback.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class Feedback(Document):
+	pass

frappe/core/doctype/feedback/test_feedback.py

@@ -0,0 +1,27 @@
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+
+import frappe
+import unittest
+
+class TestFeedback(unittest.TestCase):
+	def test_feedback_creation_updation(self):
+		from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
+		test_blog = make_test_blog()
+
+		frappe.db.sql("delete from `tabFeedback` where reference_doctype = 'Blog Post'")
+
+		from frappe.templates.includes.feedback.feedback import add_feedback, update_feedback
+		feedback = add_feedback('Blog Post', test_blog.name, 5, 'New feedback','test@test.com')
+
+		self.assertEqual(feedback.feedback, 'New feedback')
+		self.assertEqual(feedback.rating, 5)
+
+		updated_feedback = update_feedback('Blog Post', test_blog.name, 6, 'Updated feedback', 'test@test.com')
+
+		self.assertEqual(updated_feedback.feedback, 'Updated feedback')
+		self.assertEqual(updated_feedback.rating, 6)
+
+		frappe.db.sql("delete from `tabFeedback` where reference_doctype = 'Blog Post'")
+
+		test_blog.delete()

frappe/core/doctype/user/user.py

@@ -13,7 +13,7 @@ from frappe.utils.password import update_password as _update_password, check_pas
 from frappe.desk.notifications import clear_notifications
 from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings, toggle_notifications
 from frappe.utils.user import get_system_managers
-from frappe.website.utils import is_signup_enabled
+from frappe.website.utils import is_signup_disabled
 from frappe.rate_limiter import rate_limit
 from frappe.utils.background_jobs import enqueue
 from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype
@@ -839,7 +839,7 @@ def verify_password(password):
 
 @frappe.whitelist(allow_guest=True)
 def sign_up(email, full_name, redirect_to):
-	if not is_signup_enabled():
+	if is_signup_disabled():
 		frappe.throw(_('Sign Up is disabled'), title='Not Allowed')
 
 	user = frappe.db.get("User", {"email": email})
@@ -931,7 +931,7 @@ def user_query(doctype, txt, searchfield, start, page_len, filters):
 		LIMIT %(page_len)s OFFSET %(start)s
 	""".format(
 			user_type_condition = user_type_condition,
-			standard_users=", ".join([frappe.db.escape(u) for u in STANDARD_USERS]),
+			standard_users=", ".join(frappe.db.escape(u) for u in STANDARD_USERS),
 			key=searchfield,
 			fcond=get_filters_cond(doctype, filters, conditions),
 			mcond=get_match_cond(doctype)

frappe/core/doctype/user_permission/user_permission.py

@@ -16,11 +16,11 @@ class UserPermission(Document):
 		self.validate_default_permission()
 
 	def on_update(self):
-		frappe.cache().delete_value('user_permissions')
+		frappe.cache().hdel('user_permissions', self.user)
 		frappe.publish_realtime('update_user_permissions')
 
 	def on_trash(self): # pylint: disable=no-self-use
-		frappe.cache().delete_value('user_permissions')
+		frappe.cache().hdel('user_permissions', self.user)
 		frappe.publish_realtime('update_user_permissions')
 
 	def validate_user_permission(self):

frappe/core/doctype/user_type/user_type.py

@@ -112,7 +112,7 @@ class UserType(Document):
 		self.select_doctypes = []
 
 		select_doctypes = []
-		user_doctypes = tuple([row.document_type for row in self.user_doctypes])
+		user_doctypes = [row.document_type for row in self.user_doctypes]
 
 		for doctype in user_doctypes:
 			doc = frappe.get_meta(doctype)
@@ -265,4 +265,4 @@ def apply_permissions_for_non_standard_user_type(doc, method=None):
 				user_doc.update_children()
 				add_user_permission(doc.doctype, doc.name, doc.get(data[1]))
 			else:
-				frappe.db.set_value('User Permission', perm_data[0], 'user', doc.get(data[1]))
+				frappe.db.set_value('User Permission', perm_data[0], 'user', doc.get(data[1]))

frappe/custom/doctype/customize_form/customize_form.py

@@ -355,9 +355,9 @@ class CustomizeForm(Document):
 
 	def delete_custom_fields(self):
 		meta = frappe.get_meta(self.doc_type)
-		fields_to_remove = (set([df.fieldname for df in meta.get("fields")])
-			- set(df.fieldname for df in self.get("fields")))
-
+		fields_to_remove = (
+			{df.fieldname for df in meta.get("fields")} - {df.fieldname for df in self.get("fields")}
+		)
 		for fieldname in fields_to_remove:
 			df = meta.get("fields", {"fieldname": fieldname})[0]
 			if df.get("is_custom_field"):

frappe/database/database.py

@@ -335,7 +335,7 @@ class Database(object):
 				values[key] = value[1]
 				if isinstance(value[1], (tuple, list)):
 					# value is a list in tuple ("in", ("A", "B"))
-					_rhs = " ({0})".format(", ".join([self.escape(v) for v in value[1]]))
+					_rhs = " ({0})".format(", ".join(self.escape(v) for v in value[1]))
 					del values[key]
 
 			if _operator not in ["=", "!=", ">", ">=", "<", "<=", "like", "in", "not in", "not like"]:
@@ -1010,7 +1010,7 @@ class Database(object):
 			:params values: list of list of values
 		"""
 		insert_list = []
-		fields = ", ".join(["`"+field+"`" for field in fields])
+		fields = ", ".join("`"+field+"`" for field in fields)
 
 		for idx, value in enumerate(values):
 			insert_list.append(tuple(value))

frappe/desk/doctype/form_tour/form_tour.js

@@ -0,0 +1,24 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Form Tour', {
+	setup: function(frm) {
+		frm.set_query("reference_doctype", function() {
+			return {
+				filters: {
+					istable: 0
+				}
+			};
+		});
+
+		frm.set_query("field", "steps", function() {
+			return {
+				query: "frappe.desk.doctype.form_tour.form_tour.get_docfield_list",
+				filters: {
+					doctype: frm.doc.reference_doctype,
+					hidden: 0
+				}
+			};
+		});
+	}
+});

frappe/desk/doctype/form_tour/form_tour.json

@@ -0,0 +1,75 @@
+{
+ "actions": [],
+ "autoname": "field:title",
+ "creation": "2021-05-21 23:02:52.242721",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "title",
+  "reference_doctype",
+  "completed",
+  "section_break_3",
+  "steps"
+ ],
+ "fields": [
+  {
+   "fieldname": "reference_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Reference Document",
+   "options": "DocType",
+   "reqd": 1,
+   "unique": 1
+  },
+  {
+   "fieldname": "steps",
+   "fieldtype": "Table",
+   "label": "Steps",
+   "options": "Form Tour Step",
+   "reqd": 1
+  },
+  {
+   "default": "0",
+   "depends_on": "eval: doc.__islocal != 1",
+   "fieldname": "completed",
+   "fieldtype": "Check",
+   "label": "Mark as Completed"
+  },
+  {
+   "fieldname": "section_break_3",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "title",
+   "fieldtype": "Data",
+   "label": "Title",
+   "reqd": 1,
+   "unique": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-05-26 19:36:59.093753",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "Form Tour",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/desk/doctype/form_tour/form_tour.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+import frappe
+from frappe.model.document import Document
+
+class FormTour(Document):
+	pass
+
+@frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
+def get_docfield_list(doctype, txt, searchfield, start, page_len, filters):
+	or_filters = [
+		['fieldname', 'like', '%' + txt + '%'],
+		['label', 'like', '%' + txt + '%'],
+		['fieldtype', 'like', '%' + txt + '%']
+	]
+
+	parent_doctype = filters.pop('doctype')
+	excluded_fieldtypes = ['Column Break']
+	excluded_fieldtypes += filters.get('excluded_fieldtypes', [])
+
+	docfields = frappe.get_all(
+		doctype,
+		fields=["name as value", "label", "fieldtype"],
+		filters={'parent': parent_doctype, 'fieldtype': ['not in', excluded_fieldtypes]},
+		or_filters=or_filters,
+		limit_start=start,
+		limit_page_length=page_len,
+		as_list=1,
+	)
+	return docfields

frappe/desk/doctype/form_tour/test_form_tour.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+
+# import frappe
+import unittest
+
+class TestFormTour(unittest.TestCase):
+	pass

frappe/desk/doctype/form_tour_step/form_tour_step.json

@@ -0,0 +1,85 @@
+{
+ "actions": [],
+ "creation": "2021-05-21 23:05:45.342114",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "field",
+  "title",
+  "description",
+  "column_break_2",
+  "position",
+  "fieldname",
+  "label",
+  "condition"
+ ],
+ "fields": [
+  {
+   "fieldname": "title",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Title",
+   "reqd": 1
+  },
+  {
+   "columns": 4,
+   "fieldname": "description",
+   "fieldtype": "HTML Editor",
+   "in_list_view": 1,
+   "label": "Description",
+   "reqd": 1
+  },
+  {
+   "fieldname": "field",
+   "fieldtype": "Link",
+   "label": "Field",
+   "options": "DocField",
+   "reqd": 1
+  },
+  {
+   "fetch_from": "field.fieldname",
+   "fieldname": "fieldname",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Fieldname",
+   "read_only": 1
+  },
+  {
+   "fetch_from": "field.label",
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Label",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_2",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "Bottom",
+   "fieldname": "position",
+   "fieldtype": "Select",
+   "label": "Position",
+   "options": "Left\nLeft Center\nLeft Bottom\nTop\nTop Center\nTop Right\nRight\nRight Center\nRight Bottom\nBottom\nBottom Center\nBottom Right\nMid Center"
+  },
+  {
+   "fieldname": "next_step_condition",
+   "fieldtype": "Code",
+   "label": "Next Step Condition",
+   "options": "JS"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-05-26 19:44:48.737453",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "Form Tour Step",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/desk/doctype/form_tour_step/form_tour_step.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class FormTourStep(Document):
+	pass

frappe/desk/doctype/global_search_settings/global_search_settings.py

@@ -21,7 +21,7 @@ class GlobalSearchSettings(Document):
 			dts.append(dt.document_type)
 
 		if core_dts:
-			core_dts = (", ".join([frappe.bold(dt) for dt in core_dts]))
+			core_dts = ", ".join(frappe.bold(dt) for dt in core_dts)
 			frappe.throw(_("Core Modules {0} cannot be searched in Global Search.").format(core_dts))
 
 		if repeated_dts:
@@ -60,7 +60,7 @@ def update_global_search_doctypes():
 			if search_doctypes.get(domain):
 				global_search_doctypes.extend(search_doctypes.get(domain))
 
-	doctype_list = set([dt.name for dt in frappe.get_all("DocType")])
+	doctype_list = {dt.name for dt in frappe.get_all("DocType")}
 	allowed_in_global_search = []
 
 	for dt in global_search_doctypes:

frappe/desk/doctype/tag/tag.py

@@ -131,7 +131,7 @@ def update_tags(doc, tags):
 		:param doc: Document to be added to global tags
 	"""
 
-	new_tags = list(set([tag.strip() for tag in tags.split(",") if tag]))
+	new_tags = {tag.strip() for tag in tags.split(",") if tag}
 
 	for tag in new_tags:
 		if not frappe.db.exists("Tag Link", {"parenttype": doc.doctype, "parent": doc.name, "tag": tag}):
@@ -186,4 +186,4 @@ def get_documents_for_tag(tag):
 
 @frappe.whitelist()
 def get_tags_list_for_awesomebar():
-	return [t.name for t in frappe.get_list("Tag")]
+	return [t.name for t in frappe.get_list("Tag")]

frappe/desk/doctype/workspace/workspace.py

@@ -55,8 +55,7 @@ class Workspace(Document):
 		for link in self.links:
 			link = link.as_dict()
 			if link.type == "Card Break":
-
-				if card_links:
+				if card_links and (not current_card.only_for or current_card.only_for == frappe.get_system_settings('country')): 
 					current_card['links'] = card_links
 					cards.append(current_card)
 

frappe/email/doctype/email_account/email_account.py

@@ -1,32 +1,25 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # For license information, please see license.txt
-import frappe
+import email.utils
+import functools
 import imaplib
-import re
-import json
 import socket
 import time
-import functools
-
-import email.utils
-
-from frappe import _, are_emails_muted
-from frappe.model.document import Document
-from frappe.utils import (validate_email_address, cint, cstr, get_datetime,
-	DATE_FORMAT, strip, comma_or, sanitize_html, add_days, parse_addr)
-from frappe.utils.user import is_system_user
-from frappe.utils.jinja import render_template
-from frappe.email.smtp import SMTPServer
-from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError
-from poplib import error_proto
-from dateutil.relativedelta import relativedelta
 from datetime import datetime, timedelta
+from poplib import error_proto
+
+import frappe
+from frappe import _, are_emails_muted, safe_encode
 from frappe.desk.form import assign_to
-from frappe.utils.user import get_system_managers
-from frappe.utils.background_jobs import enqueue, get_jobs
-from frappe.utils.html_utils import clean_email_html
-from frappe.utils.error import raise_error_on_no_output
+from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError
+from frappe.email.smtp import SMTPServer
 from frappe.email.utils import get_port
+from frappe.model.document import Document
+from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_address
+from frappe.utils.background_jobs import enqueue, get_jobs
+from frappe.utils.error import raise_error_on_no_output
+from frappe.utils.jinja import render_template
+from frappe.utils.user import get_system_managers
 
 OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account")
 
@@ -441,10 +434,7 @@ class EmailAccount(Document):
 					if self.enable_auto_reply:
 						self.send_auto_reply(communication, mail)
 
-					attachments = []
-					if hasattr(communication, '_attachments'):
-						attachments = [d.file_name for d in communication._attachments]
-					communication.notify(attachments=attachments, fetched_from_email_account=True)
+					communication.send_email(is_inbound_mail_communcation=True)
 			except SentEmailInInboxError:
 				frappe.db.rollback()
 			except Exception:
@@ -453,6 +443,8 @@ class EmailAccount(Document):
 				if self.use_imap:
 					self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback())
 				exceptions.append(frappe.get_traceback())
+			else:
+				frappe.db.commit()
 
 		#notify if user is linked to account
 		if len(inbound_mails)>0 and not frappe.local.flags.in_test:
@@ -478,14 +470,13 @@ class EmailAccount(Document):
 			email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule)
 			messages = email_server.get_messages() or {}
 		except Exception:
-			raise
 			frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
 			return []
 
 		mails = []
 		for index, message in enumerate(messages.get("latest_messages", [])):
-			uid = messages['uid_list'][index]
-			seen_status = 1 if messages['seen_status'][uid]=='SEEN' else 0
+			uid = messages['uid_list'][index] if messages.get('uid_list') else None
+			seen_status = 1 if messages.get('seen_status', {}).get(uid)=='SEEN' else 0
 			mails.append(InboundMail(message, self, uid, seen_status))
 
 		return mails
@@ -579,8 +570,8 @@ class EmailAccount(Document):
 			email_server.update_flag(uid_list=uid_list)
 
 			# mark communication as read
-			docnames =  ",".join([ "'%s'"%flag.get("communication") for flag in flags \
-				if flag.get("action") == "Read" ])
+			docnames =  ",".join("'%s'"%flag.get("communication") for flag in flags \
+				if flag.get("action") == "Read")
 			self.set_communication_seen_status(docnames, seen=1)
 
 			# mark communication as unread
@@ -610,7 +601,6 @@ class EmailAccount(Document):
 
 
 	def append_email_to_sent_folder(self, message):
-
 		email_server = None
 		try:
 			email_server = self.get_incoming_server(in_receive=True)
@@ -624,7 +614,8 @@ class EmailAccount(Document):
 
 		if email_server.imap:
 			try:
-				email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message.encode())
+				message = safe_encode(message)
+				email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message)
 			except Exception:
 				frappe.log_error()
 

frappe/email/doctype/email_group/email_group.json

@@ -1,6 +1,7 @@
 {
  "actions": [],
  "allow_import": 1,
+ "allow_rename": 1,
  "autoname": "field:title",
  "creation": "2015-03-18 06:08:32.729800",
  "doctype": "DocType",
@@ -50,7 +51,7 @@
    "link_fieldname": "email_group"
   }
  ],
- "modified": "2020-09-24 16:41:55.286377",
+ "modified": "2021-06-15 11:25:13.556201",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Group",

frappe/email/doctype/email_queue/email_queue.py

@@ -179,7 +179,14 @@ class SendMailContext:
 		else:
 			email_status = self.is_mail_sent_to_all() and 'Sent'
 			email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent'
-			self.queue_doc.update_status(status = email_status, commit = True)
+
+			update_fields = {'status': email_status}
+			if self.email_account_doc.is_exists_in_db():
+				update_fields['email_account'] = self.email_account_doc.name
+			else:
+				update_fields['email_account'] = None
+
+			self.queue_doc.update_status(**update_fields, commit = True)
 
 	def log_exception(self, exc_type, exc_val, exc_tb):
 		if exc_type:

frappe/email/doctype/newsletter/test_newsletter.py

@@ -42,7 +42,7 @@ class TestNewsletter(unittest.TestCase):
 		email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
 		self.assertEqual(len(email_queue_list), 4)
 
-		recipients = set([e.recipients[0].recipient for e in email_queue_list])
+		recipients = {e.recipients[0].recipient for e in email_queue_list}
 		self.assertTrue(set(emails).issubset(recipients))
 
 	def test_unsubscribe(self):

frappe/email/inbox.py

@@ -18,7 +18,7 @@ def get_email_accounts(user=None):
 			"all_accounts": ""
 		}
 
-	all_accounts = ",".join([ account.get("email_account") for account in accounts ])
+	all_accounts = ",".join(account.get("email_account") for account in accounts)
 	if len(accounts) > 1:
 		email_accounts.append({
 			"email_account": all_accounts,

frappe/email/queue.py

@@ -6,15 +6,61 @@ from frappe import msgprint, _
 from frappe.utils.verified_command import get_signed_params, verify_request
 from frappe.utils import get_url, now_datetime, cint
 
-def get_emails_sent_this_month():
-	return frappe.db.sql("""
-		SELECT COUNT(*) FROM `tabEmail Queue`
-		WHERE `status`='Sent' AND EXTRACT(YEAR_MONTH FROM `creation`) = EXTRACT(YEAR_MONTH FROM NOW())
-	""")[0][0]
+def get_emails_sent_this_month(email_account=None):
+	"""Get count of emails sent from a specific email account.
 
-def get_emails_sent_today():
-	return frappe.db.sql("""SELECT COUNT(`name`) FROM `tabEmail Queue` WHERE
-		`status` in ('Sent', 'Not Sent', 'Sending') AND `creation` > (NOW() - INTERVAL '24' HOUR)""")[0][0]
+	:param email_account: name of the email account used to send mail
+
+	if email_account=None, email account filter is not applied while counting
+	"""
+	q = """
+		SELECT
+			COUNT(*)
+		FROM
+			`tabEmail Queue`
+		WHERE
+			`status`='Sent'
+			AND
+			EXTRACT(YEAR_MONTH FROM `creation`) = EXTRACT(YEAR_MONTH FROM NOW())
+	"""
+
+	q_args = {}
+	if email_account is not None:
+		if email_account:
+			q += " AND email_account = %(email_account)s"
+			q_args['email_account'] = email_account
+		else:
+			q += " AND (email_account is null OR email_account='')"
+
+	return frappe.db.sql(q, q_args)[0][0]
+
+def get_emails_sent_today(email_account=None):
+	"""Get count of emails sent from a specific email account.
+
+	:param email_account: name of the email account used to send mail
+
+	if email_account=None, email account filter is not applied while counting
+	"""
+	q = """
+		SELECT
+			COUNT(`name`)
+		FROM
+			`tabEmail Queue`
+		WHERE
+			`status` in ('Sent', 'Not Sent', 'Sending')
+			AND
+			`creation` > (NOW() - INTERVAL '24' HOUR)
+	"""
+
+	q_args = {}
+	if email_account is not None:
+		if email_account:
+			q += " AND email_account = %(email_account)s"
+			q_args['email_account'] = email_account
+		else:
+			q += " AND (email_account is null OR email_account='')"
+
+	return frappe.db.sql(q, q_args)[0][0]
 
 def get_unsubscribe_message(unsubscribe_message, expose_recipients):
 	if unsubscribe_message:

frappe/email/receive.py

@@ -738,9 +738,6 @@ class InboundMail(Email):
 		if not reference_document and self.email_account.append_to:
 			reference_document = self.match_record_by_subject_and_sender(self.email_account.append_to)
 
-			# if not reference_document:
-			# 	reference_document = Create_reference_document(self.email_account.append_to)
-
 		self._reference_document = reference_document or ''
 		return self._reference_document
 
@@ -805,7 +802,7 @@ class InboundMail(Email):
 		except frappe.DuplicateEntryError:
 			# try and find matching parent
 			parent_name = frappe.db.get_value(self.email_account.append_to,
-				{email_fileds.sender_field: email.from_email}
+				{email_fileds.sender_field: self.from_email}
 			)
 			if parent_name:
 				parent.name = parent_name

frappe/installer.py

@@ -282,10 +282,10 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False)
 
 
 def post_install(rebuild_website=False):
-	from frappe.website import render
+	from frappe.website.utils import clear_website_cache
 
 	if rebuild_website:
-		render.clear_cache()
+		clear_website_cache()
 
 	init_singles()
 	frappe.db.commit()
@@ -537,7 +537,7 @@ def is_downgrade(sql_file_path, verbose=False):
 
 def is_partial(sql_file_path):
 	with open(sql_file_path) as f:
-		header = " ".join([f.readline() for _ in range(5)])
+		header = " ".join(f.readline() for _ in range(5))
 		if "Partial Backup" in header:
 			return True
 	return False

frappe/integrations/doctype/ldap_settings/ldap_settings.py

@@ -79,7 +79,7 @@ class LDAPSettings(Document):
 
 	def sync_roles(self, user, additional_groups=None):
 
-		current_roles = set([d.role for d in user.get("roles")])
+		current_roles = set(d.role for d in user.get("roles"))
 
 		needed_roles = set()
 		needed_roles.add(self.default_role)

frappe/migrate.py

@@ -13,7 +13,7 @@ from frappe.utils.connections import check_connection
 from frappe.utils.dashboard import sync_dashboards
 from frappe.cache_manager import clear_global_cache
 from frappe.desk.notifications import clear_notifications
-from frappe.website import render
+from frappe.website.utils import clear_website_cache
 from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
@@ -76,7 +76,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 		frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu()
 
 		# syncs statics
-		render.clear_cache()
+		clear_website_cache()
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()

frappe/model/__init__.py

@@ -165,7 +165,7 @@ def delete_fields(args_dict, delete=0):
 			frappe.db.sql("""
 				DELETE FROM `tabSingles`
 				WHERE doctype='%s' AND field IN (%s)
-			""" % (dt, ", ".join(["'{}'".format(f) for f in fields])))
+			""" % (dt, ", ".join("'{}'".format(f) for f in fields)))
 		else:
 			existing_fields = frappe.db.multisql({
 					"mariadb": "DESC `tab%s`" % dt,
@@ -188,7 +188,7 @@ def delete_fields(args_dict, delete=0):
 				frappe.db.commit()
 
 			query = "ALTER TABLE `tab%s` " % dt + \
-				", ".join(["DROP COLUMN `%s`" % f for f in fields_need_to_delete])
+				", ".join("DROP COLUMN `%s`" % f for f in fields_need_to_delete)
 			frappe.db.sql(query)
 
 		if frappe.db.db_type == 'postgres':

frappe/model/base_document.py

@@ -354,7 +354,7 @@ class BaseDocument(object):
 			frappe.db.sql("""INSERT INTO `tab{doctype}` ({columns})
 					VALUES ({values})""".format(
 					doctype = self.doctype,
-					columns = ", ".join(["`"+c+"`" for c in columns]),
+					columns = ", ".join("`"+c+"`" for c in columns),
 					values = ", ".join(["%s"] * len(columns))
 				), list(d.values()))
 		except Exception as e:
@@ -397,7 +397,7 @@ class BaseDocument(object):
 			frappe.db.sql("""UPDATE `tab{doctype}`
 				SET {values} WHERE `name`=%s""".format(
 					doctype = self.doctype,
-					values = ", ".join(["`"+c+"`=%s" for c in columns])
+					values = ", ".join("`"+c+"`=%s" for c in columns)
 				), list(d.values()) + [name])
 		except Exception as e:
 			if frappe.db.is_unique_key_violation(e):

frappe/model/db_query.py

@@ -43,8 +43,14 @@ class DatabaseQuery(object):
 
 		# filters and fields swappable
 		# its hard to remember what comes first
-		if (isinstance(fields, dict)
-			or (isinstance(fields, list) and fields and isinstance(fields[0], list))):
+		if (
+			isinstance(fields, dict)
+			or (
+				fields
+				and isinstance(fields, list)
+				and isinstance(fields[0], list)
+			)
+		):
 			# if fields is given as dict/list of list, its probably filters
 			filters, fields = fields, filters
 
@@ -56,10 +62,7 @@ class DatabaseQuery(object):
 		if fields:
 			self.fields = fields
 		else:
-			if pluck:
-				self.fields =  ["`tab{0}`.`{1}`".format(self.doctype, pluck)]
-			else:
-				self.fields =  ["`tab{0}`.`name`".format(self.doctype)]
+			self.fields =  [f"`tab{self.doctype}`.`{pluck or 'name'}`"]
 
 		if start: limit_start = start
 		if page_length: limit_page_length = page_length
@@ -70,7 +73,7 @@ class DatabaseQuery(object):
 		self.docstatus = docstatus or []
 		self.group_by = group_by
 		self.order_by = order_by
-		self.limit_start = 0 if (limit_start is False) else cint(limit_start)
+		self.limit_start = cint(limit_start)
 		self.limit_page_length = cint(limit_page_length) if limit_page_length else None
 		self.with_childnames = with_childnames
 		self.debug = debug
@@ -157,11 +160,10 @@ class DatabaseQuery(object):
 
 		# left join parent, child tables
 		for child in self.tables[1:]:
-			args.tables += " {join} {child} on ({child}.parent = {main}.name)".format(join=self.join,
-				child=child, main=self.tables[0])
+			args.tables += f" {self.join} {child} on ({child}.parent = {self.tables[0]}.name)"
 
 		if self.grouped_or_conditions:
-			self.conditions.append("({0})".format(" or ".join(self.grouped_or_conditions)))
+			self.conditions.append(f"({' or '.join(self.grouped_or_conditions)})")
 
 		args.conditions = ' and '.join(self.conditions)
 
@@ -186,9 +188,9 @@ class DatabaseQuery(object):
 				fields.append(field)
 			elif "as" in field.lower().split(" "):
 				col, _, new = field.split()
-				fields.append("`{0}` as {1}".format(col, new))
+				fields.append(f"`{col}` as {new}")
 			else:
-				fields.append("`{0}`".format(field))
+				fields.append(f"`{field}`")
 
 		args.fields = ", ".join(fields)
 
@@ -260,10 +262,10 @@ class DatabaseQuery(object):
 				if any(keyword in field.lower().split() for keyword in blacklisted_keywords):
 					_raise_exception()
 
-				if any("({0}".format(keyword) in field.lower() for keyword in blacklisted_keywords):
+				if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords):
 					_raise_exception()
 
-				if any("{0}(".format(keyword) in field.lower() for keyword in blacklisted_functions):
+				if any(f"{keyword}(" in field.lower() for keyword in blacklisted_functions):
 					_raise_exception()
 
 				if '@' in field.lower():
@@ -287,22 +289,30 @@ class DatabaseQuery(object):
 
 	def extract_tables(self):
 		"""extract tables from fields"""
-		self.tables = ['`tab' + self.doctype + '`']
-
+		self.tables = [f"`tab{self.doctype}`"]
+		sql_functions = [
+			"dayofyear(",
+			"extract(",
+			"locate(",
+			"strpos(",
+			"count(",
+			"sum(",
+			"avg(",
+		]
 		# add tables from fields
 		if self.fields:
-			for f in self.fields:
-				if ( not ("tab" in f and "." in f) ) or ("locate(" in f) or ("strpos(" in f) or \
-					("count(" in f) or ("avg(" in f)  or ("sum(" in f) or ("extract(" in f) or ("dayofyear(" in f):
+			for field in self.fields:
+				if not ("tab" in field and "." in field) or any(x for x in sql_functions if x in field):
 					continue
 
-				table_name = f.split('.')[0]
+				table_name = field.split('.')[0]
+
 				if table_name.lower().startswith('group_concat('):
 					table_name = table_name[13:]
 				if table_name.lower().startswith('ifnull('):
 					table_name = table_name[7:]
 				if not table_name[0]=='`':
-					table_name = '`' + table_name + '`'
+					table_name = f"`{table_name}`"
 				if not table_name in self.tables:
 					self.append_table(table_name)
 
@@ -311,8 +321,7 @@ class DatabaseQuery(object):
 		doctype = table_name[4:-1]
 		ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read'
 
-		if (not self.flags.ignore_permissions) and\
-			 (not frappe.has_permission(doctype, ptype=ptype)):
+		if not self.flags.ignore_permissions and not frappe.has_permission(doctype, ptype=ptype):
 			frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(doctype))
 			raise frappe.PermissionError(doctype)
 
@@ -326,7 +335,7 @@ class DatabaseQuery(object):
 		if len(self.tables) > 1:
 			for idx, field in enumerate(self.fields):
 				if '.' not in field and not _in_standard_sql_methods(field):
-					self.fields[idx] = '{0}.{1}'.format(self.tables[0], field)
+					self.fields[idx] = f"{self.tables[0]}.{field}"
 
 	def get_table_columns(self):
 		try:
@@ -375,7 +384,7 @@ class DatabaseQuery(object):
 		if not self.flags.ignore_permissions:
 			match_conditions = self.build_match_conditions()
 			if match_conditions:
-				self.conditions.append("(" + match_conditions + ")")
+				self.conditions.append(f"({match_conditions})")
 
 	def build_filter_conditions(self, filters, conditions, ignore_permissions=None):
 		"""build conditions from user filters"""
@@ -407,8 +416,7 @@ class DatabaseQuery(object):
 		if 'ifnull(' in f.fieldname:
 			column_name = f.fieldname
 		else:
-			column_name = '{tname}.{fname}'.format(tname=tname,
-				fname=f.fieldname)
+			column_name = f"{tname}.{f.fieldname}"
 
 		can_be_null = True
 
@@ -450,7 +458,7 @@ class DatabaseQuery(object):
 			fallback = "''"
 			value = [frappe.db.escape((v.name or '').strip(), percent=False) for v in result]
 			if len(value):
-				value = "({0})".format(", ".join(value))
+				value = f"({', '.join(value)})"
 			else:
 				value = "('')"
 			# changing operator to IN as the above code fetches all the parent / child values and convert into tuple
@@ -466,7 +474,7 @@ class DatabaseQuery(object):
 			fallback = "''"
 			value = [frappe.db.escape((v or '').strip(), percent=False) for v in values]
 			if len(value):
-				value = "({0})".format(", ".join(value))
+				value = f"({', '.join(value)})"
 			else:
 				value = "('')"
 		else:
@@ -503,7 +511,7 @@ class DatabaseQuery(object):
 				can_be_null = True
 
 				if 'ifnull' not in column_name:
-					column_name = 'ifnull({}, {})'.format(column_name, fallback)
+					column_name = f'ifnull({column_name}, {fallback})'
 
 			elif df and df.fieldtype=="Date":
 				value = frappe.db.format_date(f.value)
@@ -540,21 +548,19 @@ class DatabaseQuery(object):
 
 			# escape value
 			if isinstance(value, str) and not f.operator.lower() == 'between':
-				value = "{0}".format(frappe.db.escape(value, percent=False))
+				value = f"{frappe.db.escape(value, percent=False)}"
 
-		if (self.ignore_ifnull
+		if (
+			self.ignore_ifnull
 			or not can_be_null
 			or (f.value and f.operator.lower() in ('=', 'like'))
-			or 'ifnull(' in column_name.lower()):
+			or 'ifnull(' in column_name.lower()
+		):
 			if f.operator.lower() == 'like' and frappe.conf.get('db_type') == 'postgres':
 				f.operator = 'ilike'
-			condition = '{column_name} {operator} {value}'.format(
-				column_name=column_name, operator=f.operator,
-				value=value)
+			condition = f'{column_name} {f.operator} {value}'
 		else:
-			condition = 'ifnull({column_name}, {fallback}) {operator} {value}'.format(
-				column_name=column_name, fallback=fallback, operator=f.operator,
-				value=value)
+			condition = f'ifnull({column_name}, {fallback}) {f.operator} {value}'
 
 		return condition
 
@@ -572,10 +578,12 @@ class DatabaseQuery(object):
 		role_permissions = frappe.permissions.get_role_permissions(meta, user=self.user)
 		self.shared = frappe.share.get_shared(self.doctype, self.user)
 
-		if (not meta.istable and
+		if (
+			not meta.istable and
 			not (role_permissions.get("select") or role_permissions.get("read")) and
 			not self.flags.ignore_permissions and
-			not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype)):
+			not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype)
+		):
 			only_if_shared = True
 			if not self.shared:
 				frappe.throw(_("No permission to read {0}").format(self.doctype), frappe.PermissionError)
@@ -585,8 +593,10 @@ class DatabaseQuery(object):
 		else:
 			#if has if_owner permission skip user perm check
 			if role_permissions.get("has_if_owner_enabled") and role_permissions.get("if_owner", {}):
-				self.match_conditions.append("`tab{0}`.`owner` = {1}".format(self.doctype,
-					frappe.db.escape(self.user, percent=False)))
+				self.match_conditions.append(
+					f"`tab{self.doctype}`.`owner` = {frappe.db.escape(self.user, percent=False)}"
+				)
+
 			# add user permission only if role has read perm
 			elif role_permissions.get("read") or role_permissions.get("select"):
 				# get user permissions
@@ -605,8 +615,7 @@ class DatabaseQuery(object):
 
 			# share is an OR condition, if there is a role permission
 			if not only_if_shared and self.shared and conditions:
-				conditions =  "({conditions}) or ({shared_condition})".format(
-					conditions=conditions, shared_condition=self.get_share_condition())
+				conditions =  f"({conditions}) or ({self.get_share_condition()})"
 
 			return conditions
 
@@ -614,8 +623,7 @@ class DatabaseQuery(object):
 			return self.match_filters
 
 	def get_share_condition(self):
-		return """`tab{0}`.name in ({1})""".format(self.doctype, ", ".join(["%s"] * len(self.shared))) % \
-			tuple([frappe.db.escape(s, percent=False) for s in self.shared])
+		return f"`tab{self.doctype}`.name in ({', '.join(frappe.db.escape(s, percent=False) for s in self.shared)})"
 
 	def add_user_permissions(self, user_permissions):
 		meta = frappe.get_meta(self.doctype)
@@ -640,9 +648,7 @@ class DatabaseQuery(object):
 				if frappe.get_system_settings("apply_strict_user_permissions"):
 					condition = ""
 				else:
-					empty_value_condition = "ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(
-						doctype=self.doctype, fieldname=df.get('fieldname')
-					)
+					empty_value_condition = f"ifnull(`tab{self.doctype}`.`{df.get('fieldname')}`, '')=''"
 					condition = empty_value_condition + " or "
 
 				for permission in user_permission_values:
@@ -650,9 +656,7 @@ class DatabaseQuery(object):
 						docs.append(permission.get('doc'))
 
 					# append docs based on user permission applicable on reference doctype
-
 					# this is useful when getting list of docs from a link field
-
 					# in this case parent doctype of the link
 					# will be the reference doctype
 
@@ -664,14 +668,9 @@ class DatabaseQuery(object):
 						docs.append(permission.get('doc'))
 
 				if docs:
-					condition += "`tab{doctype}`.`{fieldname}` in ({values})".format(
-						doctype=self.doctype,
-						fieldname=df.get('fieldname'),
-						values=", ".join(
-							[(frappe.db.escape(doc, percent=False)) for doc in docs])
-						)
-
-					match_conditions.append("({condition})".format(condition=condition))
+					values = ", ".join(frappe.db.escape(doc, percent=False) for doc in docs)
+					condition += f"`tab{self.doctype}`.`{df.get('fieldname')}` in ({values})"
+					match_conditions.append(f"({condition})")
 					match_filters[df.get('options')] = docs
 
 		if match_conditions:
@@ -721,17 +720,17 @@ class DatabaseQuery(object):
 					# `idx desc, modified desc`
 					# will covert to
 					# `tabItem`.`idx` desc, `tabItem`.`modified` desc
-					args.order_by = ', '.join(['`tab{0}`.`{1}` {2}'.format(self.doctype,
-						f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')])
+					args.order_by = ', '.join(
+						f"`tab{self.doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" for f in meta.sort_field.split(',')
+					)
 				else:
 					sort_field = meta.sort_field or 'modified'
 					sort_order = (meta.sort_field and meta.sort_order) or 'desc'
-
-					args.order_by = "`tab{0}`.`{1}` {2}".format(self.doctype, sort_field or "modified", sort_order or "desc")
+					args.order_by = f"`tab{self.doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}"
 
 				# draft docs always on top
-				if meta.is_submittable:
-					args.order_by = "`tab{0}`.docstatus asc, {1}".format(self.doctype, args.order_by)
+				if hasattr(meta, 'is_submittable') and meta.is_submittable:
+					args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}"
 
 	def validate_order_by_and_group_by(self, parameters):
 		"""Check order by, group by so that atleast one column is selected and does not have subquery"""
@@ -802,17 +801,16 @@ def get_order_by(doctype, meta):
 		# `idx desc, modified desc`
 		# will covert to
 		# `tabItem`.`idx` desc, `tabItem`.`modified` desc
-		order_by = ', '.join(['`tab{0}`.`{1}` {2}'.format(doctype,
-			f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')])
+		order_by = ', '.join(f"`tab{doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" for f in meta.sort_field.split(','))
+
 	else:
 		sort_field = meta.sort_field or 'modified'
 		sort_order = (meta.sort_field and meta.sort_order) or 'desc'
-
-		order_by = "`tab{0}`.`{1}` {2}".format(doctype, sort_field or "modified", sort_order or "desc")
+		order_by = f"`tab{doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}"
 
 	# draft docs always on top
 	if meta.is_submittable:
-		order_by = "`tab{0}`.docstatus asc, {1}".format(doctype, order_by)
+		order_by = f"`tab{doctype}`.docstatus asc, {order_by}"
 
 	return order_by
 

frappe/model/document.py

@@ -14,8 +14,8 @@ from frappe.model.workflow import set_workflow_state_on_action
 from frappe.utils.global_search import update_global_search
 from frappe.integrations.doctype.webhook import run_webhooks
 from frappe.desk.form.document_follow import follow_document
-from frappe.desk.utils import slug
 from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event
+from frappe.utils.data import get_absolute_url
 
 # once_only validation
 # methods
@@ -1200,8 +1200,8 @@ class Document(BaseDocument):
 			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield)))
 
 	def get_url(self):
-		"""Returns Desk URL for this document. `/app/{doctype}/{name}`"""
-		return f"/app/{slug(self.doctype)}/{self.name}"
+		"""Returns Desk URL for this document."""
+		return get_absolute_url(self.doctype, self.name)
 
 	def add_comment(self, comment_type='Comment', text=None, comment_email=None, link_doctype=None, link_name=None, comment_by=None):
 		"""Add a comment to this document.

frappe/model/meta.py

@@ -664,7 +664,7 @@ def trim_tables(doctype=None):
 			and not f.startswith("_")]
 		if columns_to_remove:
 			print(doctype, "columns removed:", columns_to_remove)
-			columns_to_remove = ", ".join(["drop `{0}`".format(c) for c in columns_to_remove])
+			columns_to_remove = ", ".join("drop `{0}`".format(c) for c in columns_to_remove)
 			query = """alter table `tab{doctype}` {columns}""".format(
 				doctype=doctype, columns=columns_to_remove)
 			frappe.db.sql_ddl(query)

frappe/model/rename_doc.py

@@ -141,7 +141,7 @@ def update_user_settings(old, new, link_fields):
 	if not link_fields: return
 
 	# find the user settings for the linked doctypes
-	linked_doctypes = set([d.parent for d in link_fields if not d.issingle])
+	linked_doctypes = {d.parent for d in link_fields if not d.issingle}
 	user_settings_details = frappe.db.sql('''SELECT `user`, `doctype`, `data`
 			FROM `__UserSettings`
 			WHERE `data` like %s

frappe/parallel_test_runner.py

@@ -114,13 +114,30 @@ class ParallelTestRunner():
 
 			# Generate coverage report only for app that is being tested
 			source_path = os.path.join(get_bench_path(), 'apps', self.app)
-			omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss',
-				'*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*']
+			incl = [
+				'*.py',
+			]
+			omit = [
+				'*.js',
+				'*.xml',
+				'*.pyc',
+				'*.css',
+				'*.less',
+				'*.scss',
+				'*.vue',
+				'*.pyc',
+				'*.html',
+				'*/test_*',
+				'*/node_modules/*',
+				'*/doctype/*/*_dashboard.py',
+				'*/patches/*',
+			]
 
 			if self.app == 'frappe':
+				omit.append('*/tests/*')
 				omit.append('*/commands/*')
 
-			self.coverage = Coverage(source=[source_path], omit=omit)
+			self.coverage = Coverage(source=[source_path], omit=omit, include=incl)
 			self.coverage.start()
 
 	def save_coverage(self):

frappe/patches.txt

@@ -1,11 +1,5 @@
 frappe.patches.v12_0.remove_deprecated_fields_from_doctype #3
-execute:frappe.db.sql("""update `tabPatch Log` set patch=replace(patch, '.4_0.', '.v4_0.')""") #2014-05-12
-frappe.patches.v5_0.convert_to_barracuda_and_utf8mb4
 execute:frappe.utils.global_search.setup_global_search_table()
-frappe.patches.v8_0.update_global_search_table
-frappe.patches.v7_0.update_auth
-frappe.patches.v8_0.drop_in_dialog #2017-09-22
-frappe.patches.v7_2.remove_in_filter
 execute:frappe.reload_doc('core', 'doctype', 'doctype_action', force=True) #2019-09-23
 execute:frappe.reload_doc('core', 'doctype', 'doctype_link', force=True) #2020-10-17
 execute:frappe.reload_doc('core', 'doctype', 'doctype', force=True) #2017-09-22
@@ -14,7 +8,6 @@ frappe.patches.v11_0.drop_column_apply_user_permissions
 execute:frappe.reload_doc('core', 'doctype', 'custom_docperm')
 execute:frappe.reload_doc('core', 'doctype', 'docperm') #2018-05-29
 execute:frappe.reload_doc('core', 'doctype', 'comment')
-frappe.patches.v8_0.drop_is_custom_from_docperm
 execute:frappe.reload_doc('core', 'doctype', 'document_naming_rule', force=True)
 execute:frappe.reload_doc('core', 'doctype', 'module_def') #2020-08-28
 execute:frappe.reload_doc('core', 'doctype', 'version') #2017-04-01
@@ -25,190 +18,40 @@ execute:frappe.reload_doc('core', 'doctype', 'communication') #2019-10-02
 execute:frappe.reload_doc('core', 'doctype', 'server_script')
 frappe.patches.v11_0.replicate_old_user_permissions
 frappe.patches.v11_0.reload_and_rename_view_log #2019-01-03
-frappe.patches.v7_1.rename_scheduler_log_to_error_log
-frappe.patches.v6_1.rename_file_data
-frappe.patches.v7_0.re_route #2016-06-27
-frappe.patches.v8_0.update_records_in_global_search #11-05-2017
-frappe.patches.v8_0.update_published_in_global_search
 frappe.patches.v11_0.copy_fetch_data_from_options
 frappe.patches.v11_0.change_email_signature_fieldtype
 execute:frappe.reload_doc('core', 'doctype', 'activity_log')
 execute:frappe.reload_doc('core', 'doctype', 'deleted_document')
 execute:frappe.reload_doc('core', 'doctype', 'domain_settings')
 frappe.patches.v13_0.rename_custom_client_script
-frappe.patches.v8_0.rename_page_role_to_has_role #2017-03-16
-frappe.patches.v7_2.setup_custom_perms #2017-01-19
-frappe.patches.v8_0.set_user_permission_for_page_and_report #2017-03-20
 execute:frappe.reload_doc('core', 'doctype', 'role') #2017-05-23
 execute:frappe.reload_doc('core', 'doctype', 'user') #2017-10-27
-execute:frappe.reload_doc('custom', 'doctype', 'custom_field') #2015-10-19
-execute:frappe.reload_doc('core', 'doctype', 'page') #2013-13-26
 execute:frappe.reload_doc('core', 'doctype', 'report_column')
 execute:frappe.reload_doc('core', 'doctype', 'report_filter')
 execute:frappe.reload_doc('core', 'doctype', 'report') #2020-08-25
-execute:frappe.reload_doc('core', 'doctype', 'translation') #2016-03-03
-execute:frappe.reload_doc('email', 'doctype', 'email_alert') #2014-07-15
-execute:frappe.reload_doc('desk', 'doctype', 'todo') #2014-12-31-1
-execute:frappe.reload_doc('custom', 'doctype', 'property_setter') #2014-12-31-1
-execute:frappe.reload_doc('core', 'doctype', 'patch_log') #2016-10-31
-execute:frappe.reload_doctype("File") # 2015-10-19
 execute:frappe.reload_doc('core', 'doctype', 'error_snapshot')
-execute:frappe.clear_cache()
-frappe.patches.v7_1.rename_scheduler_log_to_error_log
-frappe.patches.v7_1.sync_language_doctype
-frappe.patches.v7_0.rename_bulk_email_to_email_queue
-frappe.patches.v7_1.rename_chinese_language_codes
-
-execute:frappe.db.sql("alter table `tabSessions` modify `user` varchar(255), engine=InnoDB")
-execute:frappe.db.sql("delete from `tabDocField` where parent='0'")
-frappe.patches.v4_0.change_varchar_length
-frappe.patches.v6_4.reduce_varchar_length
-frappe.patches.v5_2.change_checks_to_not_null
-frappe.patches.v6_9.int_float_not_null #2015-11-25
-frappe.patches.v5_0.v4_to_v5
-
-frappe.patches.v5_0.remove_shopping_cart_app
-frappe.patches.v4_0.webnotes_to_frappe
-execute:frappe.permissions.reset_perms("Module Def")
-execute:import frappe.installer;frappe.installer.make_site_dirs() #2014-02-19
-frappe.patches.v4_0.rename_profile_to_user
-frappe.patches.v4_0.deprecate_control_panel
-frappe.patches.v4_0.remove_old_parent
-frappe.patches.v4_0.rename_sitemap_to_route
-frappe.patches.v4_0.website_sitemap_hierarchy
-frappe.patches.v4_0.remove_index_sitemap
-frappe.patches.v4_0.set_website_route_idx
-frappe.patches.v4_0.add_delete_permission
-frappe.patches.v4_0.set_todo_checked_as_closed
-frappe.patches.v4_0.private_backups
-frappe.patches.v4_0.set_module_in_report
-frappe.patches.v4_0.update_datetime
-frappe.patches.v4_0.file_manager_hooks
 execute:frappe.get_doc("User", "Guest").save()
-frappe.patches.v4_0.update_custom_field_insert_after
-frappe.patches.v4_0.deprecate_link_selects
-frappe.patches.v4_0.set_user_gravatar
-frappe.patches.v4_0.set_user_permissions
-frappe.patches.v4_0.create_custom_field_for_owner_match
-frappe.patches.v4_0.enable_scheduler_in_system_settings
-execute:frappe.db.sql("update tabReport set apply_user_permissions=1") #2014-06-03
-frappe.patches.v4_0.replace_deprecated_timezones
-execute:import frappe.website.render; frappe.website.render.clear_cache("login"); #2014-06-10
-frappe.patches.v4_0.fix_attach_field_file_url
-execute:frappe.permissions.reset_perms("User") #2015-03-24
-execute:frappe.db.sql("""delete from `tabUserRole` where ifnull(parentfield, '')='' or ifnull(`role`, '')=''""") #2014-08-18
-frappe.patches.v4_0.remove_user_owner_custom_field
-execute:frappe.delete_doc("DocType", "Website Template")
-execute:frappe.db.sql("""update `tabProperty Setter` set property_type='Text' where property in ('options', 'default')""") #2014-06-20
-frappe.patches.v4_1.enable_outgoing_email_settings
-execute:frappe.db.sql("""update `tabSingles` set `value`=`doctype` where `field`='name'""") #2014-07-04
-frappe.patches.v4_1.enable_print_as_pdf #2014-06-17
-execute:frappe.db.sql("""update `tabDocPerm` set email=1 where parent='User' and permlevel=0 and `role`='All' and `read`=1 and apply_user_permissions=1""") #2014-07-15
-execute:frappe.db.sql("""update `tabPrint Format` set print_format_type='Client' where ifnull(print_format_type, '')=''""") #2014-07-28
-frappe.patches.v4_1.file_manager_fix
-frappe.patches.v4_2.print_with_letterhead
 execute:frappe.delete_doc("DocType", "Control Panel", force=1)
-execute:frappe.reload_doc('website', 'doctype', 'web_form') #2014-09-04
-execute:frappe.reload_doc('website', 'doctype', 'web_form_field') #2014-09-04
-frappe.patches.v4_2.refactor_website_routing
-frappe.patches.v4_2.set_assign_in_doc
-frappe.patches.v4_3.remove_allow_on_submit_customization
-frappe.patches.v5_0.rename_table_fieldnames
-frappe.patches.v5_0.communication_parent
-frappe.patches.v5_0.clear_website_group_and_notifications
-frappe.patches.v5_0.update_shared
-execute:frappe.reload_doc("core", "doctype", "docshare") #2015-07-21
-frappe.patches.v6_19.comment_feed_communication
-frappe.patches.v6_16.star_to_like
-frappe.patches.v5_0.bookmarks_to_stars
-frappe.patches.v5_0.style_settings_to_website_theme
-frappe.patches.v5_0.rename_ref_type_fieldnames
-frappe.patches.v5_0.fix_email_alert
-frappe.patches.v5_0.fix_null_date_datetime
-frappe.patches.v5_0.force_sync_website
 execute:frappe.delete_doc("DocType", "Tag")
 execute:frappe.db.sql("delete from `tabProperty Setter` where `property` in ('idx', '_idx')")
-frappe.patches.v5_0.move_scheduler_last_event_to_system_settings
 execute:frappe.db.sql("update tabUser set new_password='' where ifnull(new_password, '')!=''")
-frappe.patches.v5_0.fix_text_editor_file_urls
-frappe.patches.v5_0.modify_session
-frappe.patches.v5_0.expire_old_scheduler_logs
 execute:frappe.permissions.reset_perms("DocType")
 execute:frappe.db.sql("delete from `tabProperty Setter` where `property` = 'idx'")
-frappe.patches.v6_0.communication_status_and_permission
-frappe.patches.v6_0.make_task_log_folder
-frappe.patches.v6_0.document_type_rename
-frappe.patches.v6_0.fix_ghana_currency
-frappe.patches.v6_2.ignore_user_permissions_if_missing
 execute:frappe.db.sql("delete from tabSessions where user is null")
-frappe.patches.v6_2.rename_backup_manager
 execute:frappe.delete_doc("DocType", "Backup Manager")
-execute:frappe.db.sql("""update `tabCommunication` set parenttype=null, parent=null, parentfield=null""") #2015-10-22
 execute:frappe.permissions.reset_perms("Web Page")
-frappe.patches.v6_6.user_last_active
-frappe.patches.v6_6.fix_file_url
-frappe.patches.v6_11.rename_field_in_email_account
-frappe.patches.v7_0.create_private_file_folder
-frappe.patches.v6_15.remove_property_setter_for_previous_field #2015-12-29
-frappe.patches.v6_15.set_username
 execute:frappe.permissions.reset_perms("Error Snapshot")
-frappe.patches.v6_16.feed_doc_owner
-frappe.patches.v6_21.print_settings_repeat_header_footer
-frappe.patches.v6_24.set_language_as_code
-frappe.patches.v6_20x.update_insert_after
-frappe.patches.v6_20x.set_allow_draft_for_print
-frappe.patches.v6_20x.remove_roles_from_website_user
-frappe.patches.v7_0.set_user_fullname
-frappe.patches.v7_0.add_communication_in_doc
-frappe.patches.v7_0.update_send_after_in_bulk_email
-execute:frappe.db.sql('''delete from `tabSingles` where doctype="Email Settings"''') # 2016-06-13
 execute:frappe.db.sql("delete from `tabWeb Page` where ifnull(template_path, '')!=''")
-frappe.patches.v7_0.rename_newsletter_list_to_email_group
-frappe.patches.v7_0.set_email_group
-frappe.patches.v7_1.setup_integration_services #2016-10-27
-frappe.patches.v7_1.rename_chinese_language_codes
 execute:frappe.core.doctype.language.language.update_language_names() # 2017-04-12
 execute:frappe.db.set_value("Print Settings", "Print Settings", "add_draft_heading", 1)
-frappe.patches.v7_0.cleanup_list_settings
 execute:frappe.db.set_default('language', '')
-frappe.patches.v7_1.refactor_integration_broker
-frappe.patches.v7_1.set_backup_limit
-frappe.patches.v7_2.set_doctype_engine
-frappe.patches.v7_2.merge_knowledge_base
-frappe.patches.v7_0.update_report_builder_json
-frappe.patches.v7_2.set_in_standard_filter_property #1
-frappe.patches.v8_0.drop_unwanted_indexes
 execute:frappe.db.sql("update tabCommunication set communication_date = creation where time(communication_date) = 0")
-frappe.patches.v7_2.fix_email_queue_recipient
-frappe.patches.v7_2.update_feedback_request # 2017-02-27
 execute:frappe.rename_doc('Country', 'Macedonia, Republic of', 'Macedonia', ignore_if_exists=True)
 execute:frappe.rename_doc('Country', 'Iran, Islamic Republic of', 'Iran', ignore_if_exists=True)
 execute:frappe.rename_doc('Country', 'Tanzania, United Republic of', 'Tanzania', ignore_if_exists=True)
 execute:frappe.rename_doc('Country', 'Syrian Arab Republic', 'Syria', ignore_if_exists=True)
-frappe.patches.v8_0.rename_listsettings_to_usersettings
-frappe.patches.v7_2.update_communications
-frappe.patches.v8_0.deprecate_integration_broker
-frappe.patches.v8_0.update_gender_and_salutation
-frappe.patches.v8_0.setup_email_inbox #2017-03-29
-frappe.patches.v8_0.newsletter_childtable_migrate
-frappe.patches.v8_0.set_doctype_values_in_custom_role
-frappe.patches.v8_0.install_new_build_system_requirements
-frappe.patches.v8_0.set_currency_field_precision	# 2017-05-09
 execute:frappe.reload_doc('desk', 'doctype', 'notification_log')
-frappe.patches.v8_0.rename_print_to_printing
-frappe.patches.v7_1.disabled_print_settings_for_custom_print_format
 execute:frappe.db.sql('update tabReport set module="Desk" where name="ToDo"')
-frappe.patches.v8_1.enable_allow_error_traceback_in_system_settings
-frappe.patches.v8_1.update_format_options_in_auto_email_report
-frappe.patches.v8_1.delete_custom_docperm_if_doctype_not_exists
-frappe.patches.v8_5.delete_email_group_member_with_invalid_emails
-frappe.patches.v8_x.update_user_permission
-frappe.patches.v8_5.patch_event_colors
-frappe.patches.v8_10.delete_static_web_page_from_global_search
-frappe.patches.v9_1.add_sms_sender_name_as_parameters
-frappe.patches.v9_1.resave_domain_settings
-frappe.patches.v9_1.revert_domain_settings
-frappe.patches.v9_1.move_feed_to_activity_log
 execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True)
 frappe.patches.v10_0.reload_countries_and_currencies # 2021-02-03
 frappe.patches.v10_0.refactor_social_login_keys

frappe/patches/v11_0/create_contact_for_user.py

@@ -8,7 +8,6 @@ def execute():
 	frappe.reload_doc('integrations', 'doctype', 'google_contacts')
 	frappe.reload_doc('contacts', 'doctype', 'contact')
 	frappe.reload_doc('core', 'doctype', 'dynamic_link')
-	frappe.reload_doc('communication', 'doctype', 'call_log')
 
 	contact_meta = frappe.get_meta("Contact")
 	if contact_meta.has_field("phone_nos") and contact_meta.has_field("email_ids"):

frappe/patches/v11_0/rename_google_maps_doctype.py

@@ -5,4 +5,3 @@ from frappe.model.rename_doc import rename_doc
 def execute():
 	if frappe.db.exists("DocType","Google Maps") and not frappe.db.exists("DocType","Google Maps Settings"):
 		rename_doc('DocType', 'Google Maps', 'Google Maps Settings')
-		frappe.reload_doc('integrations', 'doctype', 'google_maps_settings')

frappe/patches/v4_0/add_delete_permission.py

@@ -1,13 +0,0 @@
-
-import frappe
-
-def execute():
-	frappe.reload_doc("core", "doctype", "docperm")
-
-	# delete same as cancel (map old permissions)
-	frappe.db.sql("""update tabDocPerm set `delete`=ifnull(`cancel`,0)""")
-
-	# can't cancel if can't submit
-	frappe.db.sql("""update tabDocPerm set `cancel`=0 where ifnull(`submit`,0)=0""")
-
-	frappe.clear_cache()

frappe/patches/v4_0/change_varchar_length.py

@@ -1,25 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	frappe.db.sql('update tabDocField set search_index=0 where fieldtype="Small Text"')
-	frappe.db.sql('update tabDocField set in_list_view=0 where fieldtype="Image"')
-
-	for dt in frappe.db.sql_list("""select name from `tabDocType` where issingle=0"""):
-		desc = dict((d["Field"], d) for d in frappe.db.sql("desc `tab{}`".format(dt), as_dict=True))
-		alter_table = []
-
-		if desc["name"]["Type"] != "varchar(255)":
-			alter_table.append("change `name` `name` varchar(255) not null")
-
-		for fieldname in ("modified_by", "owner", "parent", "parentfield", "parenttype"):
-			if desc[fieldname]["Type"] != "varchar(255)":
-				alter_table.append("change `{fieldname}` `{fieldname}` varchar(255)".format(fieldname=fieldname))
-
-		if alter_table:
-			alter_table_query = "alter table `tab{doctype}` {alter_table}".format(doctype=dt, alter_table=",\n".join(alter_table))
-			# print alter_table_query
-			frappe.db.sql_ddl(alter_table_query)
-

frappe/patches/v4_0/create_custom_field_for_owner_match.py

@@ -1,39 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-import frappe
-from frappe.custom.doctype.custom_field.custom_field import create_custom_field
-
-def execute():
-	if "match" in frappe.db.get_table_columns("DocPerm"):
-		create_custom_field_for_owner_match()
-
-def create_custom_field_for_owner_match():
-	docperm_meta = frappe.get_meta('DocPerm')
-	if docperm_meta.get_field('apply_user_permissions'):
-		frappe.db.sql("""update `tabDocPerm` set apply_user_permissions=1 where `match`='owner'""")
-
-	for dt in frappe.db.sql_list("""select distinct parent from `tabDocPerm`
-		where `match`='owner' and permlevel=0 and parent != 'User'"""):
-
-		# a link field pointing to User already exists
-		if (frappe.db.get_value("DocField", {"parent": dt, "fieldtype": "Link", "options": "User", "default": "__user"})
-			or frappe.db.get_value("Custom Field", {"dt": dt, "fieldtype": "Link", "options": "User", "default": "__user"})):
-			print("User link field already exists for", dt)
-			continue
-
-		fieldname = "{}_owner".format(frappe.scrub(dt))
-
-		create_custom_field(dt, frappe._dict({
-			"permlevel": 0,
-			"label": "{} Owner".format(dt),
-			"fieldname": fieldname,
-			"fieldtype": "Link",
-			"options": "User",
-			"default": "__user"
-		}))
-
-		frappe.db.sql("""update `tab{doctype}` set `{fieldname}`=owner""".format(doctype=dt,
-			fieldname=fieldname))
-
-		# commit is required so that we don't lose these changes because of an error in next loop's ddl
-		frappe.db.commit()

frappe/patches/v4_0/deprecate_control_panel.py

@@ -1,9 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	frappe.db.sql("update `tabDefaultValue` set parenttype='__default' where parenttype='Control Panel'")
-	frappe.db.sql("update `tabDefaultValue` set parent='__default' where parent='Control Panel'")
-	frappe.clear_cache()

frappe/patches/v4_0/deprecate_link_selects.py

@@ -1,12 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	for name in frappe.db.sql_list("""select name from `tabCustom Field`
-		where fieldtype="Select" and options like "link:%" """):
-		custom_field = frappe.get_doc("Custom Field", name)
-		custom_field.fieldtype = "Link"
-		custom_field.options = custom_field.options[5:]
-		custom_field.save()

frappe/patches/v4_0/enable_scheduler_in_system_settings.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-from frappe.utils.scheduler import disable_scheduler, enable_scheduler
-from frappe.utils import cint
-
-def execute():
-	frappe.reload_doc("core", "doctype", "system_settings")
-	if cint(frappe.db.get_global("disable_scheduler")):
-		disable_scheduler()
-	else:
-		enable_scheduler()

frappe/patches/v4_0/file_manager_hooks.py

@@ -1,33 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-import frappe
-import os
-from frappe.core.doctype.file.file import get_content_hash
-
-
-def execute():
-	frappe.reload_doc('core', 'doctype', 'file_data')
-	for name, file_name, file_url in frappe.db.sql(
-			"""select name, file_name, file_url from `tabFile`
-			where file_name is not null"""):
-		b = frappe.get_doc('File', name)
-		old_file_name = b.file_name
-		b.file_name = os.path.basename(old_file_name)
-		if old_file_name.startswith('files/') or old_file_name.startswith('/files/'):
-			b.file_url = os.path.normpath('/' + old_file_name)
-		else:
-			b.file_url = os.path.normpath('/files/' + old_file_name)
-		try:
-			_file = frappe.get_doc("File", {"file_name": name})
-			content = _file.get_content()
-			b.content_hash = get_content_hash(content)
-		except IOError:
-			print('Warning: Error processing ', name)
-			_file_name = old_file_name
-			b.content_hash = None
-
-		try:
-			b.save()
-		except frappe.DuplicateEntryError:
-			frappe.delete_doc(b.doctype, b.name)
-

frappe/patches/v4_0/fix_attach_field_file_url.py

@@ -1,12 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	attach_fields = (frappe.db.sql("""select parent, fieldname from `tabDocField` where fieldtype in ('Attach', 'Attach Image')""") +
-		frappe.db.sql("""select dt, fieldname from `tabCustom Field` where fieldtype in ('Attach', 'Attach Image')"""))
-
-	for doctype, fieldname in attach_fields:
-		frappe.db.sql("""update `tab{doctype}` set `{fieldname}`=concat("/", `{fieldname}`)
-			where `{fieldname}` like 'files/%'""".format(doctype=doctype, fieldname=fieldname))

frappe/patches/v4_0/private_backups.py

@@ -1,10 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-from frappe.installer import make_site_dirs
-
-def execute():
-	make_site_dirs()
-	if frappe.local.conf.backup_path and frappe.local.conf.backup_path.startswith("public"):
-		raise Exception("Backups path in conf set to public directory")

frappe/patches/v4_0/remove_index_sitemap.py

@@ -1,5 +0,0 @@
-
-import frappe
-
-def execute():
-	pass

frappe/patches/v4_0/remove_old_parent.py

@@ -1,9 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	for doctype in frappe.db.sql_list("""select name from `tabDocType` where istable=1"""):
-		frappe.db.sql("""delete from `tab{0}` where parent like "old_par%:%" """.format(doctype))
-	frappe.db.sql("""delete from `tabDocField` where parent="0" """)

frappe/patches/v4_0/remove_user_owner_custom_field.py

@@ -1,9 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	user_owner = frappe.db.get_value("Custom Field", {"fieldname": "user_owner"})
-	if user_owner:
-		frappe.delete_doc("Custom Field", user_owner)

frappe/patches/v4_0/rename_profile_to_user.py

@@ -1,15 +0,0 @@
-
-import frappe
-
-from frappe.model.utils.rename_field import rename_field
-from frappe.model.meta import get_table_columns
-
-def execute():
-	tables = frappe.db.sql_list("show tables")
-	if "tabUser" not in tables:
-		frappe.rename_doc("DocType", "Profile", "User", force=True)
-
-	frappe.reload_doc("website", "doctype", "blogger")
-
-	if "profile" in get_table_columns("Blogger"):
-		rename_field("Blogger", "profile", "user")

frappe/patches/v4_0/rename_sitemap_to_route.py

@@ -1,25 +0,0 @@
-
-import frappe
-
-from frappe.model.utils.rename_field import rename_field
-
-def execute():
-	tables = frappe.db.sql_list("show tables")
-	for doctype in ("Website Sitemap", "Website Sitemap Config"):
-		if "tab{}".format(doctype) in tables:
-			frappe.delete_doc("DocType", doctype, force=1)
-			frappe.db.sql("drop table `tab{}`".format(doctype))
-
-	for d in ("Blog Category", "Blog Post", "Web Page"):
-		frappe.reload_doc("website", "doctype", frappe.scrub(d))
-		rename_field_if_exists(d, "parent_website_sitemap", "parent_website_route")
-
-	for d in ("blog_category", "blog_post", "web_page", "post", "user_vote"):
-		frappe.reload_doc("website", "doctype", d)
-
-def rename_field_if_exists(doctype, old_fieldname, new_fieldname):
-	try:
-		rename_field(doctype, old_fieldname, new_fieldname)
-	except frappe.db.ProgrammingError as e:
-		if not frappe.db.is_column_missing(e):
-			raise

frappe/patches/v4_0/replace_deprecated_timezones.py

@@ -1,20 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-from frappe.utils.momentjs import data as momentjs_data
-
-def execute():
-	frappe.reload_doc("core", "doctype", "user")
-
-	ss = frappe.get_doc("System Settings", "System Settings")
-	if ss.time_zone in momentjs_data.get("links"):
-		ss.time_zone = momentjs_data["links"][ss.time_zone]
-		ss.flags.ignore_mandatory = True
-		ss.save()
-
-	for user, time_zone in frappe.db.sql("select name, time_zone from `tabUser` where ifnull(time_zone, '')!=''"):
-		if time_zone in momentjs_data.get("links"):
-			user = frappe.get_doc("User", user)
-			user.time_zone = momentjs_data["links"][user.time_zone]
-			user.save()

frappe/patches/v4_0/set_module_in_report.py

@@ -1,9 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	frappe.reload_doc("core", "doctype", "report")
-	frappe.db.sql("""update `tabReport` r set r.module=(select d.module from `tabDocType` d
-		where d.name=r.ref_doctype) where ifnull(r.module, '')=''""")

frappe/patches/v4_0/set_todo_checked_as_closed.py

@@ -1,9 +0,0 @@
-
-import frappe
-
-def execute():
-	frappe.reload_doc("core", "doctype", "todo")
-	try:
-		frappe.db.sql("""update tabToDo set status = if(ifnull(checked,0)=0, 'Open', 'Closed')""")
-	except:
-		pass

frappe/patches/v4_0/set_user_gravatar.py

@@ -1,10 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	for name in frappe.db.sql_list("select name from `tabUser` where ifnull(user_image, '')=''"):
-		user = frappe.get_doc("User", name)
-		user.update_gravatar()
-		user.db_set("user_image", user.user_image)

frappe/patches/v4_0/set_user_permissions.py

@@ -1,23 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-import frappe.permissions
-
-def execute():
-	frappe.reload_doc("core", "doctype", "docperm")
-	table_columns = frappe.db.get_table_columns("DocPerm")
-
-	if "restricted" in table_columns:
-		frappe.db.sql("""update `tabDocPerm` set apply_user_permissions=1 where apply_user_permissions=0
-			and restricted=1""")
-
-	if "match" in table_columns:
-		frappe.db.sql("""update `tabDocPerm` set apply_user_permissions=1
-			where apply_user_permissions=0 and ifnull(`match`, '')!=''""")
-
-	# change Restriction to User Permission in tabDefaultValue
-	frappe.db.sql("""update `tabDefaultValue` set parenttype='User Permission' where parenttype='Restriction'""")
-
-	frappe.clear_cache()
-

frappe/patches/v4_0/set_website_route_idx.py

@@ -1,25 +0,0 @@
-
-import frappe
-
-def execute():
-	pass
-	# from frappe.website.doctype.website_template.website_template import \
-	# 	get_pages_and_generators, get_template_controller
-	#
-	# frappe.reload_doc("website", "doctype", "website_template")
-	# frappe.reload_doc("website", "doctype", "website_route")
-	#
-	# for app in frappe.get_installed_apps():
-	# 	pages, generators = get_pages_and_generators(app)
-	# 	for g in generators:
-	# 		doctype = frappe.get_attr(get_template_controller(app, g["path"], g["fname"]) + ".doctype")
-	# 		module = frappe.db.get_value("DocType", doctype, "module")
-	# 		frappe.reload_doc(frappe.scrub(module), "doctype", frappe.scrub(doctype))
-	#
-	# frappe.db.sql("""update `tabBlog Category` set `title`=`name` where ifnull(`title`, '')=''""")
-	# frappe.db.sql("""update `tabWebsite Route` set idx=null""")
-	# for doctype in ["Blog Category", "Blog Post", "Web Page", "Website Group"]:
-	# 	frappe.db.sql("""update `tab{}` set idx=null""".format(doctype))
-	#
-	# from frappe.website.doctype.website_template.website_template import rebuild_website_template
-	# rebuild_website_template()

frappe/patches/v4_0/update_custom_field_insert_after.py

@@ -1,17 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	for d in frappe.db.sql("""select name, dt, insert_after from `tabCustom Field`
-		where docstatus < 2""", as_dict=1):
-			dt_meta = frappe.get_meta(d.dt)
-			if not dt_meta.get_field(d.insert_after):
-				cf = frappe.get_doc("Custom Field", d.name)
-				df = dt_meta.get("fields", {"label": d.insert_after})
-				if df:
-					cf.insert_after = df[0].fieldname
-				else:
-					cf.insert_after = None
-				cf.save()

frappe/patches/v4_0/update_datetime.py

@@ -1,12 +0,0 @@
-
-import frappe
-
-def execute():
-	for table in frappe.db.sql_list("show tables"):
-		for field in frappe.db.sql("desc `%s`" % table):
-			if field[1]=="datetime":
-				frappe.db.sql("alter table `%s` change `%s` `%s` datetime(6)" % \
-					 (table, field[0], field[0]))
-			elif field[1]=="time":
-				frappe.db.sql("alter table `%s` change `%s` `%s` time(6)" % \
-					 (table, field[0], field[0]))

frappe/patches/v4_0/webnotes_to_frappe.py

@@ -1,12 +0,0 @@
-
-import frappe, json
-
-def execute():
-	frappe.clear_cache()
-	installed = frappe.get_installed_apps()
-	if "webnotes" in installed:
-		installed.remove("webnotes")
-	if "frappe" not in installed:
-		installed = ["frappe"] + installed
-	frappe.db.set_global("installed_apps", json.dumps(installed))
-	frappe.clear_cache()

frappe/patches/v4_0/website_sitemap_hierarchy.py

@@ -1,19 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	# frappe.db.sql("""update `tabWebsite Route` ws set ref_doctype=(select wsc.ref_doctype
-	# 	from `tabWebsite Template` wsc where wsc.name=ws.website_template)
-	# 	where ifnull(page_or_generator, '')!='Page'""")
-
-	frappe.reload_doc("website", "doctype", "website_settings")
-
-	# original_home_page = frappe.db.get_value("Website Settings", "Website Settings", "home_page")
-	#
-	# home_page = frappe.db.sql("""select name from `tabWebsite Route`
-	# 	where (name=%s or docname=%s) and name!='index'""", (original_home_page, original_home_page))
-	# home_page = home_page[0][0] if home_page else original_home_page
-	#
-	# frappe.db.set_value("Website Settings", "Website Settings", "home_page", home_page)

frappe/patches/v4_1/enable_outgoing_email_settings.py

@@ -1,9 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	frappe.reload_doc("core", "doctype", "outgoing_email_settings")
-	if (frappe.db.get_value("Outgoing Email Settings", "Outgoing Email Settings", "mail_server") or "").strip():
-		frappe.db.set_value("Outgoing Email Settings", "Outgoing Email Settings", "enabled", 1)

frappe/patches/v4_1/enable_print_as_pdf.py

@@ -1,25 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	frappe.reload_doc("core", "doctype", "print_settings")
-	print_settings = frappe.get_doc("Print Settings")
-	print_settings.print_style = "Modern"
-
-	try:
-		import pdfkit
-	except ImportError:
-		pass
-	else:
-		# if someone has already configured in Outgoing Email Settings
-		outgoing_email_settings = frappe.db.get_singles_dict("Outgoing Email Settings")
-		if "send_print_as_pdf" in outgoing_email_settings:
-			print_settings.send_print_as_pdf = outgoing_email_settings.send_print_as_pdf
-			print_settings.pdf_page_size = outgoing_email_settings.pdf_page_size
-
-		else:
-			print_settings.send_print_as_pdf = 1
-
-	print_settings.save()

frappe/patches/v4_1/file_manager_fix.py

@@ -1,97 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-import frappe
-import os
-from frappe.core.doctype.file.file import get_content_hash, get_file_name
-from frappe.utils import get_files_path, get_site_path
-
-# The files missed by the previous patch might have been replaced with new files
-# with the same filename
-#
-# This patch does the following,
-# * Detect which files were replaced and rename them with name{hash:5}.extn and
-#   update filedata record for the new file
-#
-# * make missing_files.txt in site dir with files that should be recovered from
-#   a backup from a time before version 3 migration
-#
-# * Patch remaining unpatched File records.
-
-
-def execute():
-	frappe.db.auto_commit_on_many_writes = True
-	rename_replacing_files()
-	for name, file_name, file_url in frappe.db.sql(
-			"""select name, file_name, file_url from `tabFile`
-			where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""):
-		b = frappe.get_doc('File', name)
-		old_file_name = b.file_name
-		b.file_name = os.path.basename(old_file_name)
-		if old_file_name.startswith('files/') or old_file_name.startswith('/files/'):
-			b.file_url = os.path.normpath('/' + old_file_name)
-		else:
-			b.file_url = os.path.normpath('/files/' + old_file_name)
-		try:
-			_file = frappe.get_doc("File", {"file_name": name})
-			content = _file.get_content()
-			b.content_hash = get_content_hash(content)
-		except IOError:
-			print('Warning: Error processing ', name)
-			b.content_hash = None
-		b.flags.ignore_duplicate_entry_error = True
-		b.save()
-	frappe.db.auto_commit_on_many_writes = False
-
-def get_replaced_files():
-	ret = []
-	new_files = dict(frappe.db.sql("select name, file_name from `tabFile` where file_name not like 'files/%'"))
-	old_files = dict(frappe.db.sql("select name, file_name from `tabFile` where ifnull(content_hash, '')=''"))
-	invfiles = invert_dict(new_files)
-
-	for nname, nfilename in new_files.items():
-		if 'files/' + nfilename in old_files.values():
-			ret.append((nfilename, invfiles[nfilename]))
-	return ret
-
-def rename_replacing_files():
-	replaced_files = get_replaced_files()
-	if len(replaced_files):
-		missing_files = [v[0] for v in replaced_files]
-		with open(get_site_path('missing_files.txt'), 'w') as f:
-			f.write(('\n'.join(missing_files) + '\n').encode('utf-8'))
-
-	for file_name, file_datas in replaced_files:
-		print ('processing ' + file_name)
-		content_hash = frappe.db.get_value('File', file_datas[0], 'content_hash')
-		if not content_hash:
-			continue
-		new_file_name = get_file_name(file_name, content_hash)
-		if os.path.exists(get_files_path(new_file_name)):
-			continue
-			print('skipping ' + file_name)
-		try:
-			os.rename(get_files_path(file_name), get_files_path(new_file_name))
-		except OSError:
-			print('Error renaming ', file_name)
-		for name in file_datas:
-			f = frappe.get_doc('File', name)
-			f.file_name = new_file_name
-			f.file_url = '/files/' + new_file_name
-			f.save()
-
-def invert_dict(ddict):
-	ret = {}
-	for k,v in ddict.items():
-		if not ret.get(v):
-			ret[v] = [k]
-		else:
-			ret[v].append(k)
-	return ret
-
-def get_file_name(fname, hash):
-		if '.' in fname:
-			partial, extn = fname.rsplit('.', 1)
-		else:
-			partial = fname
-			extn = ''
-		return '{partial}{suffix}.{extn}'.format(partial=partial, extn=extn, suffix=hash[:5])

frappe/patches/v4_2/print_with_letterhead.py

@@ -1,10 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	frappe.reload_doc("core", "doctype", "print_settings")
-	print_settings = frappe.get_doc("Print Settings")
-	print_settings.with_letterhead = 1
-	print_settings.save()

frappe/patches/v4_2/refactor_website_routing.py

@@ -1,8 +0,0 @@
-
-import frappe
-
-def execute():
-	# clear all static web pages
-	frappe.delete_doc("DocType", "Website Route", force=1)
-	frappe.delete_doc("Page", "sitemap-browser", force=1)
-	frappe.db.sql("drop table if exists `tabWebsite Route`")

frappe/patches/v4_2/set_assign_in_doc.py

@@ -1,11 +0,0 @@
-
-import frappe
-
-def execute():
-	for name in frappe.db.sql_list("""select name from `tabToDo`
-		where ifnull(reference_type, '')!='' and ifnull(reference_name, '')!=''"""):
-		try:
-			frappe.get_doc("ToDo", name).on_update()
-		except Exception as e:
-			if not frappe.db.is_table_missing(e):
-				raise

frappe/patches/v4_3/remove_allow_on_submit_customization.py

@@ -1,10 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import frappe
-
-def execute():
-	for d in frappe.get_all("Property Setter", fields=["name", "doc_type"],
-		filters={"doctype_or_field": "DocField", "property": "allow_on_submit", "value": "1"}):
-		frappe.delete_doc("Property Setter", d.name)
-		frappe.clear_cache(doctype=d.doc_type)

frappe/patches/v5_0/bookmarks_to_stars.py

@@ -1,32 +0,0 @@
-
-import json
-import frappe
-import frappe.defaults
-from frappe.desk.like import _toggle_like
-
-def execute():
-	for user in frappe.get_all("User"):
-		username = user["name"]
-		bookmarks = frappe.db.get_default("_bookmarks", username)
-
-		if not bookmarks:
-			continue
-
-		if isinstance(bookmarks, str):
-			bookmarks = json.loads(bookmarks)
-
-		for opts in bookmarks:
-			route = (opts.get("route") or "").strip("#/ ")
-
-			if route and route.startswith("Form"):
-				try:
-					view, doctype, docname = opts["route"].split("/")
-				except ValueError:
-					continue
-
-				if frappe.db.exists(doctype, docname):
-					if (doctype=="DocType"
-						or int(frappe.db.get_value("DocType", doctype, "issingle") or 0)
-						or not frappe.db.table_exists(doctype)):
-						continue
-					_toggle_like(doctype, docname, add="Yes", user=username)

frappe/patches/v5_0/clear_website_group_and_notifications.py

@@ -1,9 +0,0 @@
-
-import frappe
-
-def execute():
-	frappe.delete_doc("DocType", "Post")
-	frappe.delete_doc("DocType", "Website Group")
-	frappe.delete_doc("DocType", "Website Route Permission")
-	frappe.delete_doc("DocType", "User Vote")
-	frappe.delete_doc("DocType", "Notification Count")

frappe/patches/v5_0/communication_parent.py

@@ -1,6 +0,0 @@
-
-import frappe
-
-def execute():
-	frappe.reload_doc("core", "doctype", "communication")
-	frappe.db.sql("""update tabCommunication set reference_doctype = parenttype, reference_name = parent""")

frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py

@@ -1,16 +0,0 @@
-
-import frappe
-from frappe.database.mariadb.setup_db import check_database_settings
-from frappe.model.meta import trim_tables
-
-def execute():
-	check_database_settings()
-
-	for table in frappe.db.get_tables():
-		frappe.db.sql_ddl("""alter table `{0}` ENGINE=InnoDB ROW_FORMAT=COMPRESSED""".format(table))
-		try:
-			frappe.db.sql_ddl("""alter table `{0}` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci""".format(table))
-		except:
-			# if row size gets too large, let it be old charset!
-			pass
-

frappe/patches/v5_0/expire_old_scheduler_logs.py

@@ -1,8 +0,0 @@
-
-import frappe
-
-def execute():
-	frappe.reload_doctype("Error Log")
-
-	from frappe.core.doctype.error_log.error_log import set_old_logs_as_seen
-	set_old_logs_as_seen()

frappe/patches/v5_0/fix_email_alert.py

@@ -1,14 +0,0 @@
-import frappe
-
-def execute():
-	frappe.reload_doctype("Notification")
-	for e in frappe.get_all("Notification"):
-		notification = frappe.get_doc("Notification", e.name)
-		if notification.event == "Date Change":
-			if notification.days_in_advance < 0:
-				notification.event = "Days After"
-				notification.days_in_advance = -email_alert.days_in_advance
-			else:
-				notification.event = "Days Before"
-
-			notification.save()

frappe/patches/v5_0/fix_null_date_datetime.py

@@ -1,20 +0,0 @@
-
-import frappe
-
-def execute():
-	for table in frappe.db.get_tables():
-		changed = False
-		desc = frappe.db.sql("desc `{table}`".format(table=table), as_dict=True)
-		for field in desc:
-			if field["Type"] == "date":
-				frappe.db.sql("""update `{table}` set `{fieldname}`=null where `{fieldname}`='0000-00-00'""".format(
-					table=table, fieldname=field["Field"]))
-				changed = True
-
-			elif field["Type"] == "datetime(6)":
-				frappe.db.sql("""update `{table}` set `{fieldname}`=null where `{fieldname}`='0000-00-00 00:00:00.000000'""".format(
-					table=table, fieldname=field["Field"]))
-				changed = True
-
-		if changed:
-			frappe.db.commit()

frappe/patches/v5_0/fix_text_editor_file_urls.py

@@ -1,43 +0,0 @@
-import frappe
-import re
-
-def execute():
-	"""Fix relative urls for image src="files/" to src="/files/" in DocTypes with text editor fields"""
-	doctypes_with_text_fields = frappe.get_all("DocField", fields=["parent", "fieldname"],
-		filters={"fieldtype": "Text Editor"})
-
-	done = []
-	for opts in doctypes_with_text_fields:
-		if opts in done:
-			continue
-
-		try:
-			result = frappe.get_all(opts.parent, fields=["name", opts.fieldname])
-		except frappe.db.SQLError:
-			# bypass single tables
-			continue
-
-		for data in result:
-			old_value = data[opts.fieldname]
-			if not old_value:
-				continue
-
-			html = scrub_relative_urls(old_value)
-			if html != old_value:
-				# print_diff(html, old_value)
-				frappe.db.set_value(opts.parent, data.name, opts.fieldname, html, update_modified=False)
-
-		done.append(opts)
-
-def scrub_relative_urls(html):
-	"""prepend a slash before a relative url"""
-	try:
-		return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html)
-	except:
-		print("Error", html)
-		raise
-
-def print_diff(html, old_value):
-	import difflib
-	diff = difflib.unified_diff(old_value.splitlines(1), html.splitlines(1), lineterm='')
-	print('\n'.join(list(diff)))