Merge branch 'develop' of github.com:frappe/frappe into multistep_webforms

.github/helper/documentation.py

@@ -24,6 +24,8 @@ def docs_link_exists(body):
 					parts = parsed_url.path.split('/')
 					if len(parts) == 5 and parts[1] == "frappe" and parts[2] in docs_repos:
 						return True
+				if parsed_url.netloc in ["docs.erpnext.com", "frappeframework.com"]:
+					return True
 
 
 if __name__ == "__main__":

.github/helper/install.sh

@@ -17,7 +17,7 @@ if [ "$TYPE" == "server" ]; then
 fi
 
 if [ "$DB" == "mariadb" ];then
-      sudo apt install mariadb-client-10.3
+      sudo apt update && sudo apt install mariadb-client-10.3
       mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
       mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
 

.github/workflows/patch-mariadb-tests.yml

@@ -32,6 +32,12 @@ jobs:
         with:
           python-version: '3.9'
 
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+          check-latest: true
+
       - name: Check if build should be run
         id: check-build
         run: |

.gitignore

@@ -11,6 +11,7 @@ dist/
 frappe/docs/current
 frappe/public/dist
 .vscode
+.vs
 node_modules
 .kdev4/
 *.kdev4

CODEOWNERS

@@ -3,18 +3,18 @@
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence,
 
-*									@frappe/frappe-review-team
-templates/				@surajshetty3416
-www/							@surajshetty3416
-integrations/			@leela
-patches/					@surajshetty3416 @gavindsouza
-email/						@leela
-event_streaming/	@ruchamahabal
-data_import*			@netchampfaris
-core/							@surajshetty3416
+*                 @frappe/frappe-review-team
+templates/        @surajshetty3416
+www/              @surajshetty3416
+integrations/     @leela
+patches/          @surajshetty3416 @gavindsouza
+email/            @leela
+event_streaming/  @ruchamahabal
+data_import*      @netchampfaris
+core/             @surajshetty3416
 database          @gavindsouza
 model             @gavindsouza
-requirements.txt	@gavindsouza
-query_builder/	@gavindsouza
-commands/					@gavindsouza
+requirements.txt  @gavindsouza
+query_builder/    @gavindsouza
+commands/         @gavindsouza
 workspace         @shariquerik

codecov.yml

@@ -11,6 +11,15 @@ coverage:
         threshold: 0.5%
         flags:
           - server
+    patch:
+      default: false
+      server:
+        target: 85%
+        threshold: 0%
+        only_pulls: true
+        if_ci_failed: ignore
+        flags:
+          - server
 
 comment:
   layout: "diff, flags"

cypress/integration/control_rating.js

@@ -10,6 +10,7 @@ context('Control Rating', () => {
 			fields: [{
 				'fieldname': 'rate',
 				'fieldtype': 'Rating',
+				'options': 7
 			}]
 		});
 	}
@@ -40,4 +41,14 @@ context('Control Rating', () => {
 			.invoke('trigger', 'mouseleave')
 			.should('not.have.class', 'star-hover');
 	});
+
+	it('check number of stars in rating', () => {
+		get_dialog_with_rating();
+
+		cy.get('div.rating')
+			.first()
+			.children('svg')
+			.should('have.length', 7);
+	});
+
 });

cypress/integration/datetime.js

@@ -92,15 +92,15 @@ context('Control Date, Time and DateTime', () => {
 				date_format: 'dd.mm.yyyy',
 				time_format: 'HH:mm:ss',
 				value: '   02.12.2019 11:00:12',
-				doc_value: '2019-12-02 11:00:12',
-				input_value: '02.12.2019 11:00:12'
+				doc_value: '2019-12-02 00:30:12', // system timezone (America/New_York)
+				input_value: '02.12.2019 11:00:12' // admin timezone (Asia/Kolkata)
 			},
 			{
 				date_format: 'mm-dd-yyyy',
 				time_format: 'HH:mm',
 				value: '   12-02-2019 11:00:00',
-				doc_value: '2019-12-02 11:00:00',
-				input_value: '12-02-2019 11:00'
+				doc_value: '2019-12-02 00:30:00', // system timezone (America/New_York)
+				input_value: '12-02-2019 11:00' // admin timezone (Asia/Kolkata)
 			}
 		];
 		datetime_formats.forEach(d => {

cypress/integration/grid_keyboard_shortcut.js

@@ -0,0 +1,49 @@
+context('Grid Keyboard Shortcut', () => {
+	let total_count = 0;
+	beforeEach(() => {
+		cy.login();
+		cy.visit('/app/doctype/User');
+	});
+	before(() => {
+		cy.login();
+		cy.visit('/app/doctype/User');
+		return cy.window().its('frappe').then(frappe => {
+			frappe.db.count('DocField', {
+				filters: {
+					'parent': 'User', 'parentfield': 'fields', 'parenttype': 'DocType'
+				}
+			}).then((r) => {
+				total_count = r;
+			});
+		});
+	});
+	it('Insert new row at the end', () => {
+		cy.add_new_row_in_grid('{ctrl}{shift}{downarrow}', (cy, total_count) => {
+			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', `${total_count+1}`);
+		}, total_count);
+	});
+	it('Insert new row at the top', () => {
+		cy.add_new_row_in_grid('{ctrl}{shift}{uparrow}', (cy) => {
+			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '1');
+		});
+	});
+	it('Insert new row below', () => {
+		cy.add_new_row_in_grid('{ctrl}{downarrow}', (cy) => {
+			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '2');
+		});
+	});
+	it('Insert new row above', () => {
+		cy.add_new_row_in_grid('{ctrl}{uparrow}', (cy) => {
+			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '1');
+		});
+	});
+});
+
+Cypress.Commands.add('add_new_row_in_grid', (shortcut_keys, callbackFn, total_count) => {
+	cy.get('.frappe-control[data-fieldname="fields"]').as('table');
+	cy.get('@table').find('.grid-body .col-xs-2').first().click();
+	cy.get('@table').find('.grid-body .col-xs-2')
+		.first().type(shortcut_keys);
+
+	callbackFn(cy, total_count);
+});

cypress/integration/list_view.js

@@ -7,18 +7,13 @@ context('List View', () => {
 		});
 	});
 
-	it('Keep checkbox checked after Bulk Update', () => {
+	it('Keep checkbox checked after Refresh', () => {
 		cy.go_to_list('ToDo');
 		cy.get('.list-row-container .list-row-checkbox').click({ multiple: true, force: true });
-		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
-		cy.get('.dropdown-menu li:visible .dropdown-item .menu-item-label[data-label="Edit"]').click();
-
-		cy.get('.modal-body .form-control[data-fieldname="field"]').first().select('Priority').wait(200);
-
-		cy.get('.modal-footer .standard-actions .btn-primary').click();
-		cy.wait(500);
-
-		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
+		cy.get('.actions-btn-group button').contains('Actions').should('be.visible');
+		cy.intercept('/api/method/frappe.desk.reportview.get').as('list-refresh');
+		cy.get('button[data-original-title="Refresh"]').click();
+		cy.wait('@list-refresh');
 		cy.get('.list-row-container .list-row-checkbox:checked').should('be.visible');
 	});
 

frappe/__init__.py

@@ -28,7 +28,11 @@ from .exceptions import *
 from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader)
 from .utils.lazy_loader import lazy_import
 
-from frappe.query_builder import get_query_builder, patch_query_execute
+from frappe.query_builder import (
+	get_query_builder,
+	patch_query_execute,
+	patch_query_aggregation,
+)
 
 __version__ = '14.0.0-dev'
 
@@ -211,6 +215,7 @@ def init(site, sites_path=None, new_site=False):
 
 	setup_module_map()
 	patch_query_execute()
+	patch_query_aggregation()
 
 	local.initialised = True
 
@@ -790,7 +795,9 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc
 def is_table(doctype):
 	"""Returns True if `istable` property (indicating child Table) is set for given DocType."""
 	def get_tables():
-		return db.sql_list("select name from tabDocType where istable=1")
+		return db.get_values(
+			"DocType", filters={"istable": 1}, order_by=None, pluck=True
+		)
 
 	tables = cache().get_value("is_table", get_tables)
 	return doctype in tables

frappe/app.py

@@ -185,7 +185,9 @@ def make_form_dict(request):
 	if 'application/json' in (request.content_type or '') and request_data:
 		args = json.loads(request_data)
 	else:
-		args = request.form or request.args
+		args = {}
+		args.update(request.args or {})
+		args.update(request.form or {})
 
 	if not isinstance(args, dict):
 		frappe.throw(_("Invalid request arguments"))

frappe/boot.py

@@ -17,6 +17,7 @@ from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_p
 from frappe.model.base_document import get_controller
 from frappe.social.doctype.post.post import frequently_visited_links
 from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings, get_app_logo
+from frappe.utils import get_time_zone
 
 def get_bootinfo():
 	"""build and return boot info"""
@@ -58,6 +59,7 @@ def get_bootinfo():
 	bootinfo.home_folder = frappe.db.get_value("File", {"is_home_folder": 1})
 	bootinfo.navbar_settings = get_navbar_settings()
 	bootinfo.notification_settings = get_notification_settings()
+	set_time_zone(bootinfo)
 
 	# ipinfo
 	if frappe.session.data.get('ipinfo'):
@@ -220,8 +222,8 @@ def load_translations(bootinfo):
 	bootinfo["__messages"] = messages
 
 def get_user_info():
-	user_info = frappe.db.get_all('User', fields=['`name`', 'full_name as fullname', 'user_image as image',
-		'gender', 'email', 'username', 'bio', 'location', 'interest', 'banner_image', 'allowed_in_mentions', 'user_type'],
+	user_info = frappe.db.get_all('User', fields=['`name`', 'full_name as fullname', 'user_image as image', 'gender',
+		'email', 'username', 'bio', 'location', 'interest', 'banner_image', 'allowed_in_mentions', 'user_type', 'time_zone'],
 		filters=dict(enabled=1))
 
 	user_info_map = {d.name: d for d in user_info}
@@ -324,3 +326,9 @@ def get_desk_settings():
 
 def get_notification_settings():
 	return frappe.get_cached_doc('Notification Settings', frappe.session.user)
+
+def set_time_zone(bootinfo):
+	bootinfo.time_zone = {
+		"system": get_time_zone(),
+		"user": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) or get_time_zone()
+	}

frappe/client.py

@@ -18,7 +18,7 @@ Requests via FrappeClient are also handled here.
 
 @frappe.whitelist()
 def get_list(doctype, fields=None, filters=None, order_by=None,
-	limit_start=None, limit_page_length=20, parent=None, debug=False, as_dict=True):
+	limit_start=None, limit_page_length=20, parent=None, debug=False, as_dict=True, or_filters=None):
 	'''Returns a list of records by filters, fields, ordering and limit
 
 	:param doctype: DocType of the data to be queried
@@ -34,6 +34,7 @@ def get_list(doctype, fields=None, filters=None, order_by=None,
 		doctype=doctype,
 		fields=fields,
 		filters=filters,
+		or_filters=or_filters,
 		order_by=order_by,
 		limit_start=limit_start,
 		limit_page_length=limit_page_length,

frappe/commands/site.py

@@ -447,11 +447,10 @@ def disable_user(context, email):
 @pass_context
 def migrate(context, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
-	import re
 	from frappe.migrate import migrate
 
 	for site in context.sites:
-		print('Migrating', site)
+		click.secho(f"Migrating {site}", fg="green")
 		frappe.init(site=site)
 		frappe.connect()
 		try:
@@ -697,8 +696,7 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path=
 
 	drop_user_and_database(frappe.conf.db_name, root_login, root_password)
 
-	if not archived_sites_path:
-		archived_sites_path = os.path.join(frappe.get_app_path('frappe'), '..', '..', '..', 'archived_sites')
+	archived_sites_path = archived_sites_path or os.path.join(frappe.get_app_path('frappe'), '..', '..', '..', 'archived', 'sites')
 
 	if not os.path.exists(archived_sites_path):
 		os.mkdir(archived_sites_path)
@@ -829,39 +827,37 @@ def publish_realtime(context, event, message, room, user, doctype, docname, afte
 @pass_context
 def browse(context, site, user=None):
 	'''Opens the site on web browser'''
-	from frappe.auth import LoginManager
-	from frappe.auth import CookieManager
-	import webbrowser
+	from frappe.auth import CookieManager, LoginManager
 
-	site = context.sites[0] if context.sites else site
+	site = get_site(context, raise_err=False) or site
 
 	if not site:
-		click.echo('''Please provide site name\n\nUsage:\n\tbench browse [site-name]\nor\n\tbench --site [site-name] browse''')
-		return
+		raise SiteNotSpecifiedError
 
-	site = site.lower()
+	if site not in frappe.utils.get_sites():
+		click.echo(f"\nSite named {click.style(site, bold=True)} doesn't exist\n", err=True)
+		sys.exit(1)
 
-	if site in frappe.utils.get_sites():
-		frappe.init(site=site)
-		frappe.connect()
+	frappe.init(site=site)
+	frappe.connect()
 
-		sid = ''
-		if user:
-			if frappe.conf.developer_mode or user == "Administrator":
-				frappe.utils.set_request(path="/")
-				frappe.local.cookie_manager = CookieManager()
-				frappe.local.login_manager = LoginManager()
-				frappe.local.login_manager.login_as(user)
-				sid = f'/app?sid={frappe.session.sid}'
-			else:
-				print("Please enable developer mode to login as a user")
+	sid = ''
+	if user:
+		if frappe.conf.developer_mode or user == "Administrator":
+			frappe.utils.set_request(path="/")
+			frappe.local.cookie_manager = CookieManager()
+			frappe.local.login_manager = LoginManager()
+			frappe.local.login_manager.login_as(user)
+			sid = f'/app?sid={frappe.session.sid}'
+		else:
+			click.echo("Please enable developer mode to login as a user")
 
-		url = f'{frappe.utils.get_site_url(site)}{sid}'
-		if user == "Administrator":
-			print(f'Login URL: {url}')
-		webbrowser.open(url, new=2)
-	else:
-		click.echo("\nSite named \033[1m{}\033[0m doesn't exist\n".format(site))
+	url = f'{frappe.utils.get_site_url(site)}{sid}'
+
+	if user == "Administrator":
+		click.echo(f'Login URL: {url}')
+
+	click.launch(url)
 
 
 @click.command('start-recording')

frappe/core/doctype/communication/communication.json

@@ -51,6 +51,7 @@
   "email_inbox",
   "message_id",
   "uid",
+  "imap_folder",
   "email_status",
   "has_attachment",
   "feedback_section",
@@ -382,12 +383,19 @@
    "label": "Timeline Links",
    "options": "Communication Link",
    "permlevel": 2
+  },
+  {
+   "fieldname": "imap_folder",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "IMAP Folder",
+   "read_only": 1
   }
  ],
  "icon": "fa fa-comment",
  "idx": 1,
  "links": [],
- "modified": "2021-03-25 09:44:28.963538",
+ "modified": "2021-11-30 09:03:25.728637",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Communication",

frappe/core/doctype/communication/test_communication.py

@@ -291,6 +291,7 @@ def create_email_account():
 		"unreplied_for_mins": 20,
 		"send_notification_to": "test_comm@example.com",
 		"pop3_server": "pop.test.example.com",
+		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}],
 		"no_remaining":"0",
 		"enable_automatic_linking": 1
 	}).insert(ignore_permissions=True)

frappe/core/doctype/data_import/importer.py

@@ -199,7 +199,7 @@ class Importer:
 		new_doc = frappe.new_doc(self.doctype)
 		new_doc.update(doc)
 
-		if (meta.autoname or "").lower() != "prompt":
+		if not doc.name and (meta.autoname or "").lower() != "prompt":
 			# name can only be set directly if autoname is prompt
 			new_doc.set("name", None)
 

frappe/core/doctype/doctype/doctype.js

@@ -1,16 +1,6 @@
 // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 // MIT License. See license.txt
 
-// -------------
-// Menu Display
-// -------------
-
-// $(cur_frm.wrapper).on("grid-row-render", function(e, grid_row) {
-// 	if(grid_row.doc && grid_row.doc.fieldtype=="Section Break") {
-// 		$(grid_row.row).css({"font-weight": "bold"});
-// 	}
-// })
-
 frappe.ui.form.on('DocType', {
 	refresh: function(frm) {
 		frm.set_query('role', 'permissions', function(doc) {
@@ -129,7 +119,7 @@ frappe.ui.form.on('DocType', {
 		}
 
 		frm.set_df_property('fields', 'reqd', frm.doc.autoname !== 'Prompt');
-	}
+	},
 });
 
 frappe.ui.form.on("DocField", {
@@ -153,11 +143,10 @@ frappe.ui.form.on("DocField", {
 			curr_value.doctype = doctype;
 			curr_value.fieldname = fieldname;
 		}
-		let curr_df_link_doctype = row.fieldtype == "Link" ? row.options : null;
 
 		let doctypes = frm.doc.fields
 			.filter(df => df.fieldtype == "Link")
-			.filter(df => df.options && df.options != curr_df_link_doctype)
+			.filter(df => df.options && df.fieldname != row.fieldname)
 			.map(df => ({
 				label: `${df.options} (${df.fieldname})`,
 				value: df.fieldname
@@ -217,5 +206,11 @@ frappe.ui.form.on("DocField", {
 			$doctype_select.val(curr_value.doctype);
 			update_fieldname_options();
 		}
+	},
+
+	fieldtype: function(frm) {
+		frm.trigger("max_attachments");
 	}
 });
+
+extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm}));

frappe/core/doctype/doctype/doctype.py

@@ -75,6 +75,7 @@ class DocType(Document):
 		self.make_repeatable()
 		self.validate_nestedset()
 		self.validate_website()
+		self.ensure_minimum_max_attachment_limit()
 		validate_links_table_fieldnames(self)
 
 		if not self.is_new():
@@ -246,6 +247,22 @@ class DocType(Document):
 			# clear website cache
 			clear_cache()
 
+	def ensure_minimum_max_attachment_limit(self):
+		"""Ensure that max_attachments is *at least* bigger than number of attach fields."""
+		from frappe.model import attachment_fieldtypes
+
+
+		if not self.max_attachments:
+			return
+
+		total_attach_fields = len([d for d in self.fields if d.fieldtype in attachment_fieldtypes])
+		if total_attach_fields > self.max_attachments:
+			self.max_attachments = total_attach_fields
+			field_label = frappe.bold(self.meta.get_field("max_attachments").label)
+			frappe.msgprint(_("Number of attachment fields are more than {}, limit updated to {}.")
+					.format(field_label, total_attach_fields),
+					title=_("Insufficient attachment limit"), alert=True)
+
 	def change_modified_of_parent(self):
 		"""Change the timestamp of parent DocType if the current one is a child to clear caches."""
 		if frappe.flags.in_import:
@@ -253,7 +270,7 @@ class DocType(Document):
 		parent_list = frappe.db.get_all('DocField', 'parent',
 			dict(fieldtype=['in', frappe.model.table_fields], options=self.name))
 		for p in parent_list:
-			frappe.db.sql('UPDATE `tabDocType` SET modified=%s WHERE `name`=%s', (now(), p.parent))
+			frappe.db.update("DocType", p.parent, {}, for_update=False)
 
 	def scrub_field_names(self):
 		"""Sluggify fieldnames if not set from Label."""
@@ -1057,6 +1074,11 @@ def validate_fields(meta):
 		if getattr(docfield, 'max_height', None) and (docfield.max_height[-2:] not in ('px', 'em')):
 			frappe.throw('Max for {} height must be in px, em, rem'.format(frappe.bold(docfield.fieldname)))
 
+	def check_no_of_ratings(docfield):
+		if docfield.fieldtype == "Rating":
+			if docfield.options and (int(docfield.options) > 10 or int(docfield.options) < 3):
+				frappe.throw(_('Options for Rating field can range from 3 to 10'))
+
 	fields = meta.get("fields")
 	fieldname_list = [d.fieldname for d in fields]
 
@@ -1090,6 +1112,7 @@ def validate_fields(meta):
 		scrub_fetch_from(d)
 		validate_data_field_type(d)
 		check_max_height(d)
+		check_no_of_ratings(d)
 
 	check_fold(fields)
 	check_search_fields(meta, fields)

frappe/core/doctype/doctype_state/doctype_state.json

@@ -0,0 +1,50 @@
+{
+ "actions": [],
+ "creation": "2021-08-23 17:21:28.345841",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "title",
+  "color",
+  "custom"
+ ],
+ "fields": [
+  {
+   "fieldname": "title",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Title",
+   "reqd": 1
+  },
+  {
+   "default": "Blue",
+   "fieldname": "color",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Color",
+   "options": "Blue\nCyan\nGray\nGreen\nLight Blue\nOrange\nPink\nPurple\nRed\nYellow",
+   "reqd": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "custom",
+   "fieldtype": "Check",
+   "hidden": 1,
+   "label": "Custom"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-12-14 14:14:55.716378",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "DocType State",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}

frappe/core/doctype/doctype_state/doctype_state.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class DocTypeState(Document):
+	pass

frappe/core/doctype/file/file.py

@@ -569,6 +569,24 @@ class File(Document):
 		frappe.local.rollback_observers.append(self)
 		self.save()
 
+	@staticmethod
+	def zip_files(files):
+		from six import string_types
+
+		zip_file = io.BytesIO()
+		zf = zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED)
+		for _file in files:
+			if isinstance(_file, string_types):
+				_file = frappe.get_doc("File", _file)
+			if not isinstance(_file, File):
+				continue
+			if _file.is_folder:
+				continue
+			zf.writestr(_file.file_name, _file.get_content())
+		zf.close()
+		return zip_file.getvalue()
+
+
 def on_doctype_update():
 	frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"])
 
@@ -612,6 +630,16 @@ def move_file(file_list, new_parent, old_parent):
 	frappe.get_doc("File", old_parent).save()
 	frappe.get_doc("File", new_parent).save()
 
+
+@frappe.whitelist()
+def zip_files(files):
+	files = frappe.parse_json(files)
+	zipped_files = File.zip_files(files)
+	frappe.response["filename"] = "files.zip"
+	frappe.response["filecontent"] = zipped_files
+	frappe.response["type"] = "download"
+
+
 def setup_folder_path(filename, new_parent):
 	file = frappe.get_doc("File", filename)
 	file.folder = new_parent
@@ -940,20 +968,14 @@ def get_files_by_search_text(text):
 
 def update_existing_file_docs(doc):
 	# Update is private and file url of all file docs that point to the same file
-	frappe.db.sql("""
-		UPDATE `tabFile`
-		SET
-			file_url = %(file_url)s,
-			is_private = %(is_private)s
-		WHERE
-			content_hash = %(content_hash)s
-			and name != %(file_name)s
-	""", dict(
-		file_url=doc.file_url,
-		is_private=doc.is_private,
-		content_hash=doc.content_hash,
-		file_name=doc.name
-	))
+	file_doctype = frappe.qb.DocType("File")
+	(
+		frappe.qb.update(file_doctype)
+		.set(file_doctype.file_url, doc.file_url)
+		.set(file_doctype.is_private, doc.is_private)
+		.where(file_doctype.content_hash == doc.content_hash)
+		.where(file_doctype.name != doc.name)
+	).run()
 
 def attach_files_to_document(doc, event):
 	""" Runs on on_update hook of all documents.

frappe/core/doctype/module_profile/module_profile.js

@@ -1,19 +1,23 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Module Profile', {
-	refresh: function(frm) {
+frappe.ui.form.on("Module Profile", {
+	refresh: function (frm) {
 		if (has_common(frappe.user_roles, ["Administrator", "System Manager"])) {
 			if (!frm.module_editor && frm.doc.__onload && frm.doc.__onload.all_modules) {
-				let module_area = $('<div style="min-height: 300px">')
-					.appendTo(frm.fields_dict.module_html.wrapper);
-
+				const module_area = $(frm.fields_dict.module_html.wrapper);
 				frm.module_editor = new frappe.ModuleEditor(frm, module_area);
 			}
 		}
 
 		if (frm.module_editor) {
-			frm.module_editor.refresh();
+			frm.module_editor.show();
+		}
+	},
+
+	validate: function (frm) {
+		if (frm.module_editor) {
+			frm.module_editor.set_modules_in_table();
 		}
 	}
 });

frappe/core/doctype/module_profile/module_profile.json

@@ -34,11 +34,17 @@
   }
  ],
  "index_web_pages_for_search": 1,
- "links": [],
- "modified": "2021-01-03 15:36:52.622696",
+ "links": [
+  {
+   "link_doctype": "User",
+   "link_fieldname": "module_profile"
+  }
+ ],
+ "modified": "2021-12-03 15:47:21.296443",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Module Profile",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {

frappe/core/doctype/role_profile/role_profile.json

@@ -1,175 +1,80 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "role_profile", 
- "beta": 0, 
- "creation": "2017-08-31 04:16:38.764465", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "autoname": "role_profile",
+ "creation": "2017-08-31 04:16:38.764465",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "role_profile",
+  "roles_html",
+  "roles"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "role_profile", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Role Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
+   "fieldname": "role_profile",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Role Name",
+   "reqd": 1,
    "unique": 1
-  }, 
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "roles_html", 
-   "fieldtype": "HTML", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Roles HTML", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "roles_html",
+   "fieldtype": "HTML",
+   "label": "Roles HTML",
+   "read_only": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "roles", 
-   "fieldtype": "Table", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Roles Assigned", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Has Role", 
-   "permlevel": 1, 
-   "precision": "", 
-   "print_hide": 1, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "roles",
+   "fieldtype": "Table",
+   "hidden": 1,
+   "label": "Roles Assigned",
+   "options": "Has Role",
+   "permlevel": 1,
+   "print_hide": 1,
+   "read_only": 1
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-10-17 11:05:11.183066", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "Role Profile", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "links": [
+  {
+   "link_doctype": "User",
+   "link_fieldname": "role_profile_name"
+  }
+ ],
+ "modified": "2021-12-03 15:45:45.270963",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Role Profile",
+ "naming_rule": "Expression (old style)",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
-  }, 
+  },
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 1, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "email": 1,
+   "export": 1,
+   "permlevel": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "title_field": "role_profile", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "title_field": "role_profile",
+ "track_changes": 1
 }

frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py

@@ -10,7 +10,7 @@ from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 class TestScheduledJobType(unittest.TestCase):
 	def setUp(self):
 		frappe.db.rollback()
-		frappe.db.sql('truncate `tabScheduled Job Type`')
+		frappe.db.truncate("Scheduled Job Type")
 		sync_jobs()
 		frappe.db.commit()
 

frappe/core/doctype/server_script/server_script_utils.py

@@ -41,7 +41,19 @@ def run_server_script_for_doc_event(doc, event):
 	if scripts:
 		# run all scripts for this doctype + event
 		for script_name in scripts:
-			frappe.get_doc('Server Script', script_name).execute_doc(doc)
+			try:
+				frappe.get_doc('Server Script', script_name).execute_doc(doc)
+			except Exception as e:
+				message = frappe._('Error executing Server Script {0}. Open Browser Console to see traceback.').format(
+					frappe.utils.get_link_to_form('Server Script', script_name)
+				)
+				exception = type(e)
+				if getattr(frappe, 'request', None):
+					# all exceptions throw 500 which is internal server error
+					# however server script error is a user error
+					# so we should throw 417 which is expectation failed
+					exception.http_status_code = 417
+				frappe.throw(title=frappe._('Server Script Error'), msg=message, exc=exception)
 
 def get_server_script_map():
 	# fetch cached server script methods

frappe/core/doctype/server_script/test_server_script.py

@@ -76,7 +76,7 @@ class TestServerScript(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
 		frappe.db.commit()
-		frappe.db.sql('truncate `tabServer Script`')
+		frappe.db.truncate("Server Script")
 		frappe.get_doc('User', 'Administrator').add_roles('Script Manager')
 		for script in scripts:
 			script_doc = frappe.get_doc(doctype ='Server Script')
@@ -88,7 +88,7 @@ class TestServerScript(unittest.TestCase):
 	@classmethod
 	def tearDownClass(cls):
 		frappe.db.commit()
-		frappe.db.sql('truncate `tabServer Script`')
+		frappe.db.truncate("Server Script")
 		frappe.cache().delete_value('server_script_map')
 
 	def setUp(self):

frappe/core/doctype/system_settings/system_settings.js

@@ -32,5 +32,11 @@ frappe.ui.form.on("System Settings", {
 				frm.set_value('prepared_report_expiry_period', 7);
 			}
 		}
+	},
+	on_update: function(frm) {
+		if (frappe.boot.time_zone && frappe.boot.time_zone.system !== frm.doc.time_zone) {
+			// Clear cache after saving to refresh the values of boot.
+			frappe.ui.toolbar.clear_cache();
+		}
 	}
 });

frappe/core/doctype/system_settings/system_settings.json

@@ -66,7 +66,9 @@
   "attach_view_link",
   "prepared_report_section",
   "enable_prepared_report_auto_deletion",
-  "prepared_report_expiry_period"
+  "prepared_report_expiry_period",
+  "system_updates_section",
+  "disable_system_update_notification"
  ],
  "fields": [
   {
@@ -95,6 +97,7 @@
    "fieldname": "time_zone",
    "fieldtype": "Select",
    "label": "Time Zone",
+   "read_only": 1,
    "reqd": 1
   },
   {
@@ -462,12 +465,24 @@
    "fieldname": "encrypt_backup",
    "fieldtype": "Check",
    "label": "Encrypt Backups"
+  },
+  {
+   "collapsible": 1,
+   "fieldname": "system_updates_section",
+   "fieldtype": "Section Break",
+   "label": "System Updates"
+  },
+  {
+   "default": "0",
+   "fieldname": "disable_system_update_notification",
+   "fieldtype": "Check",
+   "label": "Disable System Update Notification"
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2021-10-21 19:24:15.232430",
+ "modified": "2021-11-29 18:09:53.601629",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

frappe/core/doctype/transaction_log/transaction_log.py

@@ -9,6 +9,7 @@ from frappe.model.document import Document
 from frappe.query_builder import DocType
 from frappe.utils import cint, now_datetime
 
+
 class TransactionLog(Document):
 	def before_insert(self):
 		index = get_current_index()
@@ -29,18 +30,15 @@ class TransactionLog(Document):
 	def hash_line(self):
 		sha = hashlib.sha256()
 		sha.update(
-			frappe.safe_encode(str(self.row_index)) + \
-			frappe.safe_encode(str(self.timestamp)) + \
-			frappe.safe_encode(str(self.data))
+			frappe.safe_encode(str(self.row_index))
+			+ frappe.safe_encode(str(self.timestamp))
+			+ frappe.safe_encode(str(self.data))
 		)
 		return sha.hexdigest()
 
 	def hash_chain(self):
 		sha = hashlib.sha256()
-		sha.update(
-			frappe.safe_encode(str(self.transaction_hash)) + \
-			frappe.safe_encode(str(self.previous_hash))
-		)
+		sha.update(frappe.safe_encode(str(self.transaction_hash)) + frappe.safe_encode(str(self.previous_hash)))
 		return sha.hexdigest()
 
 

frappe/core/doctype/user/test_user.py

@@ -251,7 +251,7 @@ class TestUser(unittest.TestCase):
 		c = FrappeClient(url)
 		res1 = c.session.post(url, data=data, verify=c.verify, headers=c.headers)
 		res2 = c.session.post(url, data=data, verify=c.verify, headers=c.headers)
-		self.assertEqual(res1.status_code, 200)
+		self.assertEqual(res1.status_code, 400)
 		self.assertEqual(res2.status_code, 417)
 
 	def test_user_rename(self):

frappe/core/doctype/user/user.js

@@ -50,7 +50,7 @@ frappe.ui.form.on('User', {
 						let d = frm.add_child("block_modules");
 						d.module = v.module;
 					});
-					frm.module_editor && frm.module_editor.refresh();
+					frm.module_editor && frm.module_editor.show();
 				}
 			});
 		}
@@ -77,7 +77,12 @@ frappe.ui.form.on('User', {
 		}
 	},
 	refresh: function(frm) {
-		var doc = frm.doc;
+		let doc = frm.doc;
+
+		if (frm.is_new()) {
+			frm.set_value("time_zone", frappe.sys_defaults.time_zone);
+		}
+
 		if (in_list(['System User', 'Website User'], frm.doc.user_type)
 			&& !frm.is_new() && !frm.roles_editor && frm.can_edit_roles) {
 			frm.reload_doc();
@@ -180,7 +185,7 @@ frappe.ui.form.on('User', {
 				frm.roles_editor.show();
 			}
 
-			frm.module_editor && frm.module_editor.refresh();
+			frm.module_editor && frm.module_editor.show();
 
 			if(frappe.session.user==doc.name) {
 				// update display settings
@@ -267,6 +272,12 @@ frappe.ui.form.on('User', {
 				}
 			}
 		});
+	},
+	on_update: function(frm) {
+		if (frappe.boot.time_zone && frappe.boot.time_zone.user !== frm.doc.time_zone) {
+			// Clear cache after saving to refresh the values of boot.
+			frappe.ui.toolbar.clear_cache();
+		}
 	}
 });
 

frappe/core/doctype/user/user.py

@@ -7,7 +7,7 @@ import frappe.defaults
 import frappe.permissions
 from frappe.model.document import Document
 from frappe.utils import (cint, flt, has_gravatar, escape_html, format_datetime,
-	now_datetime, get_formatted_email, today)
+	now_datetime, get_formatted_email, today, get_time_zone)
 from frappe import throw, msgprint, _
 from frappe.utils.password import update_password as _update_password, check_password, get_password_reset_limit
 from frappe.desk.notifications import clear_notifications
@@ -74,6 +74,7 @@ class User(Document):
 		self.validate_roles()
 		self.validate_allowed_modules()
 		self.validate_user_image()
+		self.set_time_zone()
 
 		if self.language == "Loading...":
 			self.language = None
@@ -213,15 +214,12 @@ class User(Document):
 		user_type_doc.update_modules_in_user(self)
 
 	def has_desk_access(self):
-		'''Return true if any of the set roles has desk access'''
+		"""Return true if any of the set roles has desk access"""
 		if not self.roles:
 			return False
 
-		return len(frappe.db.sql("""select name
-			from `tabRole` where desk_access=1
-				and name in ({0}) limit 1""".format(', '.join(['%s'] * len(self.roles))),
-				[d.role for d in self.roles]))
-
+		role_table = DocType("Role")
+		return frappe.db.count(role_table, ((role_table.desk_access == 1) & (role_table.name.isin([d.role for d in self.roles]))))
 
 	def share_with_self(self):
 		frappe.share.add(self.doctype, self.name, self.name, write=1, share=1,
@@ -230,11 +228,11 @@ class User(Document):
 	def validate_share(self, docshare):
 		pass
 		# if docshare.user == self.name:
-		# 	if self.user_type=="System User":
-		# 		if docshare.share != 1:
-		# 			frappe.throw(_("Sorry! User should have complete access to their own record."))
-		# 	else:
-		# 		frappe.throw(_("Sorry! Sharing with Website User is prohibited."))
+		#	if self.user_type=="System User":
+		#		if docshare.share != 1:
+		#			frappe.throw(_("Sorry! User should have complete access to their own record."))
+		#	else:
+		#		frappe.throw(_("Sorry! Sharing with Website User is prohibited."))
 
 	def send_password_notification(self, new_password):
 		try:
@@ -279,12 +277,20 @@ class User(Document):
 		return link
 
 	def get_other_system_managers(self):
-		return frappe.db.sql("""select distinct `user`.`name` from `tabHas Role` as `user_role`, `tabUser` as `user`
-			where user_role.role='System Manager'
-				and `user`.docstatus<2
-				and `user`.enabled=1
-				and `user_role`.parent = `user`.name
-			and `user_role`.parent not in ('Administrator', %s) limit 1""", (self.name,))
+		user_doctype = DocType("User").as_("user")
+		user_role_doctype = DocType("Has Role").as_("user_role")
+		return (
+			frappe.qb.from_(user_doctype)
+			.from_(user_role_doctype)
+			.select(user_doctype.name)
+			.where(user_role_doctype.role == 'System Manager')
+			.where(user_doctype.docstatus < 2)
+			.where(user_doctype.enabled == 1)
+			.where(user_role_doctype.parent == user_doctype.name)
+			.where(user_role_doctype.parent.notin(["Administrator", self.name]))
+			.limit(1)
+			.distinct()
+		).run()
 
 	def get_fullname(self):
 		"""get first_name space last_name"""
@@ -358,8 +364,12 @@ class User(Document):
 
 		# delete todos
 		frappe.db.delete("ToDo", {"owner": self.name})
-		frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""",
-			(self.name,))
+		todo_table = DocType("ToDo")
+		(
+			frappe.qb.update(todo_table)
+			.set(todo_table.assigned_by, None)
+			.where(todo_table.assigned_by == self.name)
+		).run()
 
 		# delete events
 		frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"})
@@ -425,10 +435,7 @@ class User(Document):
 			frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False)
 
 		# set email
-		table = DocType("User")
-		frappe.qb.update(table).where(
-			table.name == new_name
-		).set("email", new_name).run()
+		frappe.db.update("User", new_name, "email", new_name)
 
 	def append_roles(self, *roles):
 		"""Add roles to user"""
@@ -590,6 +597,10 @@ class User(Document):
 
 		return user
 
+	def set_time_zone(self):
+		if not self.time_zone:
+			self.time_zone = get_time_zone()
+
 @frappe.whitelist()
 def get_timezones():
 	import pytz
@@ -698,28 +709,19 @@ def has_email_account(email):
 
 @frappe.whitelist(allow_guest=False)
 def get_email_awaiting(user):
-	waiting = frappe.db.sql("""select email_account,email_id
-		from `tabUser Email`
-		where awaiting_password = 1
-		and parent = %(user)s""", {"user":user}, as_dict=1)
+	waiting = frappe.get_all("User Email", fields=["email_account", "email_id"], filters={"awaiting_password": 1, "parent": user})
 	if waiting:
 		return waiting
 	else:
-		frappe.db.sql("""update `tabUser Email`
-				set awaiting_password =0
-				where parent = %(user)s""",{"user":user})
+		user_email_table = DocType("User Email")
+		frappe.qb.update(user_email_table).set(user_email_table.user_email_table, 0).where(user_email_table.parent == user).run()
 		return False
 
 def ask_pass_update():
 	# update the sys defaults as to awaiting users
 	from frappe.utils import set_default
 
-	doctype = DocType("User Email")
-	users = frappe.qb.from_(doctype).where(doctype.awaiting_password == 1).select(
-		doctype.parent.as_("user")
-	).distinct().run(as_dict=True)
-
-	password_list = [ user.get("user") for user in users ]
+	password_list = frappe.get_all("User Email", filters={"awaiting_password": True}, pluck="parent", distinct=True)
 	set_default("email_user_password", u','.join(password_list))
 
 def _get_user_for_update_password(key, old_password):
@@ -811,6 +813,7 @@ def reset_password(user):
 		return frappe.msgprint(_("Password reset instructions have been sent to your email"))
 
 	except frappe.DoesNotExistError:
+		frappe.local.response['http_status_code'] = 400
 		frappe.clear_messages()
 		return 'not found'
 
@@ -887,8 +890,7 @@ def get_active_users():
 
 def get_website_users():
 	"""Returns total no. of website users"""
-	return frappe.db.sql("""select count(*) from `tabUser`
-		where enabled = 1 and user_type = 'Website User'""")[0][0]
+	return frappe.db.count("User", filters={"enabled": True, "user_type": "Website User"})
 
 def get_active_website_users():
 	"""Returns No. of website users who logged in, in the last 3 days"""

frappe/core/form_tour/doctype/doctype.json

@@ -0,0 +1,56 @@
+{
+ "creation": "2021-11-23 12:38:52.807353",
+ "docstatus": 0,
+ "doctype": "Form Tour",
+ "first_document": 0,
+ "idx": 0,
+ "include_name_field": 1,
+ "is_standard": 1,
+ "modified": "2021-11-25 17:03:01.646360",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Doctype",
+ "owner": "Administrator",
+ "reference_doctype": "DocType",
+ "save_on_complete": 1,
+ "steps": [
+  {
+   "description": "Select a Module to which this DocType would belong",
+   "field": "",
+   "fieldname": "module",
+   "fieldtype": "Link",
+   "has_next_condition": 0,
+   "is_table_field": 0,
+   "label": "Module",
+   "parent_field": "",
+   "position": "Right",
+   "title": "Module"
+  },
+  {
+   "description": "Check this to make the DocType as Custom",
+   "field": "",
+   "fieldname": "custom",
+   "fieldtype": "Check",
+   "has_next_condition": 1,
+   "is_table_field": 0,
+   "label": "Custom?",
+   "next_step_condition": "eval: doc.custom",
+   "parent_field": "",
+   "position": "Left",
+   "title": "Custom "
+  },
+  {
+   "description": "A Field (or a docfield) defines a property of a DocType. You can define the column name, label, datatype and more for DocFields. For instance, a ToDo doctype has fields description, status and priority. These ultimately become columns in the database table tabToDo.",
+   "field": "",
+   "fieldname": "fields",
+   "fieldtype": "Table",
+   "has_next_condition": 0,
+   "is_table_field": 0,
+   "label": "Fields",
+   "parent_field": "",
+   "position": "Top",
+   "title": "Fields"
+  }
+ ],
+ "title": "Doctype"
+}

frappe/core/notifications.py

@@ -2,6 +2,9 @@
 # License: MIT. See LICENSE
 
 import frappe
+from frappe.query_builder import DocType, Interval
+from frappe.query_builder.functions import Now
+
 
 def get_notification_config():
 	return {
@@ -39,28 +42,40 @@ def get_todays_events(as_list=False):
 
 def get_unseen_likes():
 	"""Returns count of unseen likes"""
-	return frappe.db.sql("""select count(*) from `tabComment`
-		where
-			comment_type='Like'
-			and modified >= (NOW() - INTERVAL '1' YEAR)
-			and owner is not null and owner!=%(user)s
-			and reference_owner=%(user)s
-			and seen=0
-			""", {"user": frappe.session.user})[0][0]
+
+	comment_doctype = DocType("Comment")
+	return frappe.db.count(comment_doctype,
+		filters=(
+			(comment_doctype.comment_type == "Like")
+			& (comment_doctype.modified >= Now() - Interval(years=1))
+			& (comment_doctype.owner.notnull())
+			& (comment_doctype.owner != frappe.session.user)
+			& (comment_doctype.reference_owner == frappe.session.user)
+			& (comment_doctype.seen == 0)
+		)
+	)
+
 
 def get_unread_emails():
-	"returns unread emails for a user"
+	"returns count of unread emails for a user"
 
-	return frappe.db.sql("""\
-		SELECT count(*)
-		FROM `tabCommunication`
-		WHERE communication_type='Communication'
-		AND communication_medium='Email'
-		AND sent_or_received='Received'
-		AND email_status not in ('Spam', 'Trash')
-		AND email_account in (
-			SELECT distinct email_account from `tabUser Email` WHERE parent=%(user)s
+	communication_doctype = DocType("Communication")
+	user_doctype = DocType("User")
+	distinct_email_accounts = (
+		frappe.qb.from_(user_doctype)
+		.select(user_doctype.email_account)
+		.where(user_doctype.parent == frappe.session.user)
+		.distinct()
+	)
+
+	return frappe.db.count(communication_doctype,
+		filters=(
+			(communication_doctype.communication_type == "Communication")
+			& (communication_doctype.communication_medium == "Email")
+			& (communication_doctype.sent_or_received == "Received")
+			& (communication_doctype.email_status.notin(["spam", "Trash"]))
+			& (communication_doctype.email_account.isin(distinct_email_accounts))
+			& (communication_doctype.modified >= Now() - Interval(years=1))
+			& (communication_doctype.seen == 0)
 		)
-		AND modified >= (NOW() - INTERVAL '1' YEAR)
-		AND seen=0
-		""", {"user": frappe.session.user})[0][0]
+	)

frappe/custom/doctype/custom_field/custom_field.py

@@ -8,6 +8,7 @@ from frappe import _
 from frappe.model.document import Document
 from frappe.model.docfield import supports_translation
 from frappe.model import core_doctypes_list
+from frappe.query_builder.functions import IfNull
 
 class CustomField(Document):
 	def autoname(self):
@@ -115,9 +116,7 @@ def get_fields_label(doctype=None):
 def create_custom_field_if_values_exist(doctype, df):
 	df = frappe._dict(df)
 	if df.fieldname in frappe.db.get_table_columns(doctype) and \
-		frappe.db.sql("""select count(*) from `tab{doctype}`
-			where ifnull({fieldname},'')!=''""".format(doctype=doctype, fieldname=df.fieldname))[0][0]:
-
+		frappe.db.count(dt=doctype, filters=IfNull(df.fieldname, "") != ""):
 		create_custom_field(doctype, df)
 
 def create_custom_field(doctype, df, ignore_validate=False):

frappe/custom/doctype/customize_form/customize_form.js

@@ -114,6 +114,7 @@ frappe.ui.form.on("Customize Form", {
 		frm.page.clear_icons();
 
 		if (frm.doc.doc_type) {
+			frm.page.set_title(__('Customize Form - {0}', [frm.doc.doc_type]));
 			frappe.customize_form.set_primary_action(frm);
 
 			frm.add_custom_button(
@@ -276,6 +277,21 @@ frappe.ui.form.on("DocType Action", {
 	}
 });
 
+// can't delete standard states
+frappe.ui.form.on("DocType State", {
+	before_states_remove: function(frm, doctype, name) {
+		let row = frappe.get_doc(doctype, name);
+		if (!(row.custom || row.__islocal)) {
+			frappe.msgprint(__("Cannot delete standard document state."));
+			throw "cannot delete standard document state";
+		}
+	},
+	states_add: function(frm, cdt, cdn) {
+		let f = frappe.model.get_doc(cdt, cdn);
+		f.custom = 1;
+	}
+});
+
 frappe.customize_form.set_primary_action = function(frm) {
 	frm.page.set_primary_action(__("Update"), function() {
 		if (frm.doc.doc_type) {
@@ -332,3 +348,4 @@ frappe.customize_form.clear_locals_and_refresh = function(frm) {
 	frm.refresh();
 }
 
+extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm}));

frappe/custom/doctype/customize_form/customize_form.json

@@ -41,6 +41,8 @@
   "actions",
   "document_links_section",
   "links",
+  "document_states_section",
+  "states",
   "section_break_8",
   "sort_field",
   "column_break_10",
@@ -280,6 +282,20 @@
    "fieldname": "autoname",
    "fieldtype": "Data",
    "label": "Auto Name"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "states",
+   "depends_on": "doc_type",
+   "fieldname": "document_states_section",
+   "fieldtype": "Section Break",
+   "label": "Document States"
+  },
+  {
+   "fieldname": "states",
+   "fieldtype": "Table",
+   "label": "States",
+   "options": "DocType State"
   }
  ],
  "hide_toolbar": 1,
@@ -288,10 +304,11 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2021-06-21 19:01:06.920663",
+ "modified": "2021-12-14 16:45:04.308690",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form",
+ "naming_rule": "Expression (old style)",
  "owner": "Administrator",
  "permissions": [
   {
@@ -308,5 +325,6 @@
  "search_fields": "doc_type",
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/custom/doctype/customize_form/customize_form.py

@@ -72,7 +72,7 @@ class CustomizeForm(Document):
 				new_d[prop] = d.get(prop)
 			self.append("fields", new_d)
 
-		for fieldname in ('links', 'actions'):
+		for fieldname in ('links', 'actions', 'states'):
 			for d in meta.get(fieldname):
 				self.append(fieldname, d)
 
@@ -258,7 +258,8 @@ class CustomizeForm(Document):
 		'''
 		for doctype, fieldname, field_map in (
 				('DocType Link', 'links', doctype_link_properties),
-				('DocType Action', 'actions', doctype_action_properties)
+				('DocType Action', 'actions', doctype_action_properties),
+				('DocType State', 'states', doctype_state_properties),
 			):
 			has_custom = False
 			items = []
@@ -568,6 +569,11 @@ doctype_action_properties = {
 	'hidden': 'Check'
 }
 
+doctype_state_properties = {
+	'title': 'Data',
+	'color': 'Select'
+}
+
 
 ALLOWED_FIELDTYPE_CHANGE = (
 	('Currency', 'Float', 'Percent'),

frappe/custom/doctype/property_setter/property_setter.json

@@ -37,7 +37,7 @@
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Applied On",
-   "options": "\nDocField\nDocType\nDocType Link\nDocType Action",
+   "options": "\nDocField\nDocType\nDocType Link\nDocType Action\nDocType State",
    "read_only_depends_on": "eval:!doc.__islocal",
    "reqd": 1
   },
@@ -109,7 +109,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-09-04 12:46:17.860769",
+ "modified": "2021-12-14 14:15:41.929071",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Property Setter",
@@ -141,5 +141,6 @@
  "search_fields": "doc_type,property",
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/custom/form_tour/custom_field/custom_field.json

@@ -0,0 +1,79 @@
+{
+ "creation": "2021-11-23 12:22:32.922700",
+ "docstatus": 0,
+ "doctype": "Form Tour",
+ "first_document": 0,
+ "idx": 0,
+ "include_name_field": 0,
+ "is_standard": 1,
+ "modified": "2021-11-24 19:15:34.244244",
+ "modified_by": "Administrator",
+ "module": "Custom",
+ "name": "Custom Field",
+ "owner": "Administrator",
+ "reference_doctype": "Custom Field",
+ "save_on_complete": 1,
+ "steps": [
+  {
+   "description": "Select a Document for which you want the Custom Field",
+   "field": "",
+   "fieldname": "dt",
+   "fieldtype": "Link",
+   "has_next_condition": 0,
+   "is_table_field": 0,
+   "label": "Document",
+   "parent_field": "",
+   "position": "Right",
+   "title": "Document"
+  },
+  {
+   "description": "Enter a Label for this field",
+   "field": "",
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "has_next_condition": 0,
+   "is_table_field": 0,
+   "label": "Label",
+   "parent_field": "",
+   "position": "Right",
+   "title": "Label"
+  },
+  {
+   "description": "Select the label after which you want to insert new field.",
+   "field": "",
+   "fieldname": "insert_after",
+   "fieldtype": "Select",
+   "has_next_condition": 0,
+   "is_table_field": 0,
+   "label": "Insert After",
+   "parent_field": "",
+   "position": "Right",
+   "title": "Insert After"
+  },
+  {
+   "description": "Select an appropriate Field Type that suits your requirements",
+   "field": "",
+   "fieldname": "fieldtype",
+   "fieldtype": "Select",
+   "has_next_condition": 0,
+   "is_table_field": 0,
+   "label": "Field Type",
+   "parent_field": "",
+   "position": "Left",
+   "title": "Field Type"
+  },
+  {
+   "description": "Check this to make it a mandatory field",
+   "field": "",
+   "fieldname": "reqd",
+   "fieldtype": "Check",
+   "has_next_condition": 0,
+   "is_table_field": 0,
+   "label": "Is Mandatory Field",
+   "parent_field": "",
+   "position": "Left",
+   "title": "Is Mandatory Field"
+  }
+ ],
+ "title": "Custom Field"
+}

frappe/custom/module_onboarding/customization/customization.json

@@ -0,0 +1,44 @@
+{
+ "allow_roles": [
+  {
+   "role": "All"
+  }
+ ],
+ "creation": "2021-11-23 12:21:11.384229",
+ "docstatus": 0,
+ "doctype": "Module Onboarding",
+ "documentation_url": "https://docs.erpnext.com/docs/v13/user/manual/en/customize-erpnext",
+ "idx": 0,
+ "is_complete": 0,
+ "modified": "2021-11-24 17:04:31.523715",
+ "modified_by": "Administrator",
+ "module": "Custom",
+ "name": "Customization",
+ "owner": "Administrator",
+ "steps": [
+  {
+   "step": "Custom Field"
+  },
+  {
+   "step": "Custom Doctype"
+  },
+  {
+   "step": "Naming Series"
+  },
+  {
+   "step": "Workflows"
+  },
+  {
+   "step": "Role Permissions"
+  },
+  {
+   "step": "Print Format"
+  },
+  {
+   "step": "Report Builder"
+  }
+ ],
+ "subtitle": "Custom Field, Custom Doctype, Naming Series, Role Permission, Workflow, Print Formats, Reports",
+ "success_message": "Customization onboarding is all done!",
+ "title": "Customization"
+}

frappe/custom/onboarding_step/custom_doctype/custom_doctype.json

@@ -0,0 +1,21 @@
+{
+ "action": "Create Entry",
+ "action_label": "Learn more about creating new DocTypes",
+ "creation": "2021-11-23 12:30:04.407568",
+ "description": "A DocType (Document Type) is used to insert forms in ERPNext. Forms such as Customer, Orders, and Invoices are Doctypes in the backend. You can also create new DocTypes to create new forms in ERPNext as per your business needs.",
+ "docstatus": 0,
+ "doctype": "Onboarding Step",
+ "idx": 0,
+ "is_complete": 0,
+ "is_single": 0,
+ "is_skipped": 0,
+ "modified": "2021-11-23 12:30:04.407568",
+ "modified_by": "Administrator",
+ "name": "Custom Doctype",
+ "owner": "Administrator",
+ "reference_document": "DocType",
+ "show_form_tour": 1,
+ "show_full_form": 1,
+ "title": "Custom Document Types",
+ "validate_action": 1
+}

frappe/custom/onboarding_step/custom_field/custom_field.json

@@ -0,0 +1,21 @@
+{
+ "action": "Create Entry",
+ "action_label": "Learn how to add Custom Fields",
+ "creation": "2021-11-23 12:21:09.479808",
+ "description": "Every form in ERPNext has a standard set of fields. If you need to capture some information, but there is no standard Field available for it, you can insert Custom Field for it.\n\nOnce custom fields are added, you can use them for reports and analytics charts as well.\n",
+ "docstatus": 0,
+ "doctype": "Onboarding Step",
+ "idx": 0,
+ "is_complete": 0,
+ "is_single": 0,
+ "is_skipped": 0,
+ "modified": "2021-11-23 12:21:09.479808",
+ "modified_by": "Administrator",
+ "name": "Custom Field",
+ "owner": "Administrator",
+ "reference_document": "Custom Field",
+ "show_form_tour": 1,
+ "show_full_form": 1,
+ "title": "Create Custom Fields",
+ "validate_action": 1
+}

frappe/custom/onboarding_step/naming_series/naming_series.json

@@ -0,0 +1,20 @@
+{
+ "action": "Watch Video",
+ "creation": "2021-11-23 13:57:45.091427",
+ "description": "Each document created in ERPNext can have a unique ID generated for it, using a prefix defined for it. Though each document has some prefix pre-configured, you can further customize it using tools like Naming Series Tool and Document Naming Rule.\n",
+ "docstatus": 0,
+ "doctype": "Onboarding Step",
+ "idx": 0,
+ "is_complete": 0,
+ "is_single": 0,
+ "is_skipped": 0,
+ "modified": "2021-11-24 15:04:14.662684",
+ "modified_by": "Administrator",
+ "name": "Naming Series",
+ "owner": "Administrator",
+ "show_form_tour": 0,
+ "show_full_form": 0,
+ "title": "Setup Naming Series",
+ "validate_action": 1,
+ "video_url": "https://youtu.be/IGyISSfI1qU"
+}

frappe/custom/onboarding_step/print_format/print_format.json

@@ -0,0 +1,21 @@
+{
+ "action": "Create Entry",
+ "action_label": "Learn about Standard and Custom Print Formats",
+ "creation": "2021-11-23 15:04:12.728513",
+ "description": "Print Formats allow you can define looks for documents when printed or converted to PDF. You can also create a custom Print Format using drag-and-drop tools.",
+ "docstatus": 0,
+ "doctype": "Onboarding Step",
+ "idx": 0,
+ "is_complete": 0,
+ "is_single": 0,
+ "is_skipped": 0,
+ "modified": "2021-11-23 15:04:12.728513",
+ "modified_by": "Administrator",
+ "name": "Print Format",
+ "owner": "Administrator",
+ "reference_document": "Print Format",
+ "show_form_tour": 1,
+ "show_full_form": 1,
+ "title": "Customize Print Formats",
+ "validate_action": 1
+}

frappe/custom/onboarding_step/report_builder/report_builder.json

@@ -0,0 +1,22 @@
+{
+ "action": "Watch Video",
+ "action_label": "Learn more about Report Builders",
+ "creation": "2021-11-24 17:04:18.762838",
+ "description": "In each module, you will find a host of single-click reports, ranging from financial statements to sales and purchase analytics and stock tracking reports. If a required new report is not available out-of-the-box, you can create custom reports in ERPNext by pulling values from the same multiple ERPNext tables.\n",
+ "docstatus": 0,
+ "doctype": "Onboarding Step",
+ "idx": 0,
+ "is_complete": 0,
+ "is_single": 0,
+ "is_skipped": 0,
+ "modified": "2021-11-24 17:04:18.762838",
+ "modified_by": "Administrator",
+ "name": "Report Builder",
+ "owner": "Administrator",
+ "reference_document": "Report",
+ "show_form_tour": 0,
+ "show_full_form": 0,
+ "title": "Generate Custom Reports",
+ "validate_action": 1,
+ "video_url": "https://youtu.be/TxJGUNarcQs"
+}

frappe/custom/onboarding_step/role_permissions/role_permissions.json

@@ -0,0 +1,20 @@
+{
+ "action": "Watch Video",
+ "creation": "2021-11-23 14:00:27.208500",
+ "description": "In ERPNext, you can add your Employees as Users, and give them restricted access. Tools like Role Permission and User Permission allow you to define rules which give restricted access to the user to masters and transactions.",
+ "docstatus": 0,
+ "doctype": "Onboarding Step",
+ "idx": 0,
+ "is_complete": 0,
+ "is_single": 0,
+ "is_skipped": 0,
+ "modified": "2021-11-24 15:04:14.615232",
+ "modified_by": "Administrator",
+ "name": "Role Permissions",
+ "owner": "Administrator",
+ "show_form_tour": 0,
+ "show_full_form": 0,
+ "title": "Setup Limited Access for a User",
+ "validate_action": 1,
+ "video_url": "https://youtu.be/g3mk45o1zAg"
+}

frappe/custom/onboarding_step/workflows/workflows.json

@@ -0,0 +1,20 @@
+{
+ "action": "Watch Video",
+ "creation": "2021-11-23 13:58:58.530044",
+ "description": "Workflows allow you to define custom rules for the approval process of a particular document in ERPNext. You can also set complex Workflow Rules and set approval conditions.",
+ "docstatus": 0,
+ "doctype": "Onboarding Step",
+ "idx": 0,
+ "is_complete": 0,
+ "is_single": 0,
+ "is_skipped": 0,
+ "modified": "2021-11-24 15:04:14.632144",
+ "modified_by": "Administrator",
+ "name": "Workflows",
+ "owner": "Administrator",
+ "show_form_tour": 0,
+ "show_full_form": 0,
+ "title": "Setup Approval Workflows",
+ "validate_action": 1,
+ "video_url": "https://youtu.be/yObJUg9FxFs"
+}

frappe/custom/workspace/customization/customization.json

@@ -1,6 +1,6 @@
 {
  "charts": [],
- "content": "[{\"type\": \"header\", \"data\": {\"text\": \"Your Shortcuts\", \"level\": 4, \"col\": 12}}, {\"type\": \"shortcut\", \"data\": {\"shortcut_name\": \"Customize Form\", \"col\": 4}}, {\"type\": \"shortcut\", \"data\": {\"shortcut_name\": \"Custom Role\", \"col\": 4}}, {\"type\": \"shortcut\", \"data\": {\"shortcut_name\": \"Client Script\", \"col\": 4}}, {\"type\": \"shortcut\", \"data\": {\"shortcut_name\": \"Server Script\", \"col\": 4}}, {\"type\": \"spacer\", \"data\": {\"col\": 12}}, {\"type\": \"header\", \"data\": {\"text\": \"Reports & Masters\", \"level\": 4, \"col\": 12}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Dashboards\", \"col\": 4}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Form Customization\", \"col\": 4}}, {\"type\": \"card\", \"data\": {\"card_name\": \"Other\", \"col\": 4}}]",
+ "content": "[{\"type\":\"onboarding\",\"data\":{\"onboarding_name\":\"Customization\",\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\\n\\t\\t\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\",\"level\":4,\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Customize Form\",\"col\":4}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Custom Role\",\"col\":4}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Client Script\",\"col\":4}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Server Script\",\"col\":4}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\\n\\t\\t\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\",\"level\":4,\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Dashboards\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Form Customization\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Other\",\"col\":4}}]",
  "creation": "2020-03-02 15:15:03.839594",
  "docstatus": 0,
  "doctype": "Workspace",
@@ -123,7 +123,7 @@
    "type": "Link"
   }
  ],
- "modified": "2021-08-05 12:15:57.486113",
+ "modified": "2021-11-24 16:20:03.500885",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customization",

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py

@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
+# Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
 import frappe
@@ -8,6 +7,20 @@ from frappe.modules.export_file import export_to_files, create_init_py
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.model.document import Document
 
+
+def get_mapping_module(module, mapping_name):
+	app_name = frappe.db.get_value("Module Def", module, "app_name")
+	mapping_name = frappe.scrub(mapping_name)
+	module = frappe.scrub(module)
+
+	try:
+		return frappe.get_module(
+			f"{app_name}.{module}.data_migration_mapping.{mapping_name}"
+		)
+	except ImportError:
+		return None
+
+
 class DataMigrationPlan(Document):
 	def on_update(self):
 		# update custom fields in mappings
@@ -54,26 +67,14 @@ class DataMigrationPlan(Document):
 		frappe.flags.ignore_in_install = False
 
 	def pre_process_doc(self, mapping_name, doc):
-		module = self.get_mapping_module(mapping_name)
+		module = get_mapping_module(self.module, mapping_name)
 
 		if module and hasattr(module, 'pre_process'):
 			return module.pre_process(doc)
 		return doc
 
 	def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None):
-		module = self.get_mapping_module(mapping_name)
+		module = get_mapping_module(self.module, mapping_name)
 
 		if module and hasattr(module, 'post_process'):
 			return module.post_process(local_doc=local_doc, remote_doc=remote_doc)
-
-	def get_mapping_module(self, mapping_name):
-		try:
-			module_def = frappe.get_doc("Module Def", self.module)
-			module = frappe.get_module('{app}.{module}.data_migration_mapping.{mapping_name}'.format(
-				app= module_def.app_name,
-				module=frappe.scrub(self.module),
-				mapping_name=frappe.scrub(mapping_name)
-			))
-			return module
-		except ImportError:
-			return None

frappe/database/database.py

@@ -171,10 +171,10 @@ class Database(object):
 				frappe.errprint(query)
 
 			elif self.is_deadlocked(e):
-				raise frappe.QueryDeadlockError
+				raise frappe.QueryDeadlockError(e)
 
 			elif self.is_timedout(e):
-				raise frappe.QueryTimeoutError
+				raise frappe.QueryTimeoutError(e)
 
 			if ignore_ddl and (self.is_missing_column(e) or self.is_missing_table(e) or self.cant_drop_field_or_key(e)):
 				pass
@@ -260,6 +260,7 @@ class Database(object):
 		self.commit()
 		self.sql(query, debug=debug)
 
+
 	def check_transaction_status(self, query):
 		"""Raises exception if more than 20,000 `INSERT`, `UPDATE` queries are
 		executed in one transaction. This is to ensure that writes are always flushed otherwise this
@@ -334,8 +335,21 @@ class Database(object):
 		"""Returns `get_value` with fieldname='*'"""
 		return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache)
 
-	def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False,
-		debug=False, order_by=None, cache=False, for_update=False, run=True):
+	def get_value(
+		self,
+		doctype,
+		filters=None,
+		fieldname="name",
+		ignore=None,
+		as_dict=False,
+		debug=False,
+		order_by="KEEP_DEFAULT_ORDERING",
+		cache=False,
+		for_update=False,
+		run=True,
+		pluck=False,
+		distinct=False,
+	):
 		"""Returns a document property or list of properties.
 
 		:param doctype: DocType name.
@@ -362,7 +376,7 @@ class Database(object):
 		"""
 
 		ret = self.get_values(doctype, filters, fieldname, ignore, as_dict, debug,
-			order_by, cache=cache, for_update=for_update, run=run)
+			order_by, cache=cache, for_update=for_update, run=run, pluck=pluck, distinct=distinct)
 
 		if not run:
 			return ret
@@ -370,7 +384,8 @@ class Database(object):
 		return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None
 
 	def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False,
-		debug=False, order_by=None, update=None, cache=False, for_update=False, run=True):
+		debug=False, order_by="KEEP_DEFAULT_ORDERING", update=None, cache=False, for_update=False,
+		run=True, pluck=False, distinct=False):
 		"""Returns multiple document properties.
 
 		:param doctype: DocType name.
@@ -379,7 +394,8 @@ class Database(object):
 		:param ignore: Don't raise exception if table, column is missing.
 		:param as_dict: Return values as dict.
 		:param debug: Print query in error log.
-		:param order_by: Column to order by
+		:param order_by: Column to order by,
+		:param distinct: Get Distinct results.
 
 		Example:
 
@@ -394,9 +410,20 @@ class Database(object):
 			(doctype, filters, fieldname) in self.value_cache:
 			return self.value_cache[(doctype, filters, fieldname)]
 
+		if distinct:
+			order_by = None
+
 		if isinstance(filters, list):
-			order_by = order_by or "modified_desc"
-			out = self._get_value_for_many_names(doctype, filters, fieldname, debug=debug, run=run)
+			out = self._get_value_for_many_names(
+				doctype,
+				filters,
+				fieldname,
+				order_by,
+				debug=debug,
+				run=run,
+				pluck=pluck,
+				distinct=distinct,
+			)
 
 		else:
 			fields = fieldname
@@ -408,9 +435,20 @@ class Database(object):
 
 			if (filters is not None) and (filters!=doctype or doctype=="DocType"):
 				try:
-					order_by = order_by or "modified"
+					if order_by:
+						order_by = "modified" if order_by == "KEEP_DEFAULT_ORDERING" else order_by
 					out = self._get_values_from_table(
-						fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run
+						fields,
+						filters,
+						doctype,
+						as_dict,
+						debug,
+						order_by,
+						update,
+						for_update=for_update,
+						run=run,
+						pluck=pluck,
+						distinct=distinct
 					)
 				except Exception as e:
 					if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)):
@@ -418,19 +456,30 @@ class Database(object):
 						out = None
 					elif (not ignore) and frappe.db.is_table_missing(e):
 						# table not found, look in singles
-						out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run)
+						out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run, distinct=distinct)
 
 					else:
 						raise
 			else:
-				out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run)
+				out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run, pluck=pluck, distinct=distinct)
 
 		if cache and isinstance(filters, str):
 			self.value_cache[(doctype, filters, fieldname)] = out
 
 		return out
 
-	def get_values_from_single(self, fields, filters, doctype, as_dict=False, debug=False, update=None, run=True):
+	def get_values_from_single(
+		self,
+		fields,
+		filters,
+		doctype,
+		as_dict=False,
+		debug=False,
+		update=None,
+		run=True,
+		pluck=False,
+		distinct=False,
+	):
 		"""Get values from `tabSingles` (Single DocTypes) (internal).
 
 		:param fields: List of fields,
@@ -456,10 +505,13 @@ class Database(object):
 				return [map(values.get, fields)]
 
 		else:
-			r = self.sql("""select field, value
-				from `tabSingles` where field in (%s) and doctype=%s"""
-					% (', '.join(['%s'] * len(fields)), '%s'),
-					tuple(fields) + (doctype,), as_dict=False, debug=debug, run=run)
+			r = self.query.get_sql(
+				"Singles",
+				filters={"field": ("in", tuple(fields)), "doctype": doctype},
+				fields=["field", "value"],
+				distinct=distinct,
+			).run(pluck=pluck, debug=debug, as_dict=False)
+
 			if not run:
 				return r
 			if as_dict:
@@ -484,14 +536,10 @@ class Database(object):
 			# Get coulmn and value of the single doctype Accounts Settings
 			account_settings = frappe.db.get_singles_dict("Accounts Settings")
 		"""
-		result = self.sql("""
-			SELECT field, value
-			FROM   `tabSingles`
-			WHERE  doctype = %s
-		""", doctype)
-
+		result = self.query.get_sql(
+			"Singles", filters={"doctype": doctype}, fields=["field", "value"]
+		).run()
 		dict_  = frappe._dict(result)
-
 		return dict_
 
 	@staticmethod
@@ -520,8 +568,11 @@ class Database(object):
 		if fieldname in self.value_cache[doctype]:
 			return self.value_cache[doctype][fieldname]
 
-		val = self.sql("""select `value` from
-			`tabSingles` where `doctype`=%s and `field`=%s""", (doctype, fieldname))
+		val = self.query.get_sql(
+			table="Singles",
+			filters={"doctype": doctype, "field": fieldname},
+			fields="value",
+		).run()
 		val = val[0][0] if val else None
 
 		df = frappe.get_meta(doctype).get_field(fieldname)
@@ -539,40 +590,64 @@ class Database(object):
 		"""Alias for get_single_value"""
 		return self.get_single_value(*args, **kwargs)
 
-	def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None,
-								update=None, for_update=False, run=True):
+	def _get_values_from_table(
+		self,
+		fields,
+		filters,
+		doctype,
+		as_dict,
+		debug,
+		order_by=None,
+		update=None,
+		for_update=False,
+		run=True,
+		pluck=False,
+		distinct=False,
+	):
 		field_objects = []
 
 		if not isinstance(fields, Criterion):
 			for field in fields:
-				if "(" in field or " as " in field:
+				if "(" in str(field) or " as " in str(field):
 					field_objects.append(PseudoColumn(field))
 				else:
 					field_objects.append(field)
 
-		criterion = self.query.build_conditions(
-			table=doctype, filters=filters, orderby=order_by, for_update=for_update
+		query = self.query.get_sql(
+			table=doctype,
+			filters=filters,
+			orderby=order_by,
+			for_update=for_update,
+			field_objects=field_objects,
+			fields=fields,
+			distinct=distinct,
 		)
-		if isinstance(fields, (list, tuple)):
-			query = criterion.select(*field_objects)
+		if (
+			fields == "*"
+			and not isinstance(fields, (list, tuple))
+			and not isinstance(fields, Criterion)
+		):
+			as_dict = True
 
-		elif isinstance(fields, Criterion):
-			query = criterion.select(fields)
-
-		else:
-			if fields=="*":
-				query = criterion.select(fields)
-				as_dict = True
-		r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run)
+		r = self.sql(
+			query, as_dict=as_dict, debug=debug, update=update, run=run, pluck=pluck
+		)
 		return r
 
-	def _get_value_for_many_names(self, doctype, names, field, debug=False, run=True):
+	def _get_value_for_many_names(self, doctype, names, field, order_by, debug=False, run=True, pluck=False, distinct=False):
 		names = list(filter(None, names))
 		if names:
-			return self.get_all(doctype,
+			return self.get_all(
+				doctype,
 				fields=field,
 				filters=names,
-				debug=debug, as_list=1, run=run)
+				order_by=order_by,
+				pluck=pluck,
+				debug=debug,
+				as_list=1,
+				run=run,
+				distinct=distinct,
+			)
 		else:
 			return {}
 
@@ -788,25 +863,13 @@ class Database(object):
 			except Exception:
 				return None
 
-	def min(self, dt, fieldname, filters=None, **kwargs):
-		return self.query.build_conditions(dt, filters=filters).select(Min(Column(fieldname))).run(**kwargs)[0][0] or 0
-
-	def max(self, dt, fieldname, filters=None, **kwargs):
-		return self.query.build_conditions(dt, filters=filters).select(Max(Column(fieldname))).run(**kwargs)[0][0] or 0
-
-	def avg(self, dt, fieldname, filters=None, **kwargs):
-		return self.query.build_conditions(dt, filters=filters).select(Avg(Column(fieldname))).run(**kwargs)[0][0] or 0
-
-	def sum(self, dt, fieldname, filters=None, **kwargs):
-		return self.query.build_conditions(dt, filters=filters).select(Sum(Column(fieldname))).run(**kwargs)[0][0] or 0
-
 	def count(self, dt, filters=None, debug=False, cache=False):
 		"""Returns `COUNT(*)` for given DocType and filters."""
 		if cache and not filters:
 			cache_count = frappe.cache().get_value('doctype:count:{}'.format(dt))
 			if cache_count is not None:
 				return cache_count
-		query = self.query.build_conditions(table=dt, filters=filters).select(Count("*"))
+		query = self.query.get_sql(table=dt, filters=filters, fields=Count("*"))
 		if filters:
 			count = self.sql(query, debug=debug)[0][0]
 			return count

frappe/database/mariadb/database.py

@@ -43,7 +43,7 @@ class MariaDBDatabase(Database):
 			'Dynamic Link':	('varchar', self.VARCHAR_LEN),
 			'Password':		('text', ''),
 			'Select':		('varchar', self.VARCHAR_LEN),
-			'Rating':		('int', '1'),
+			'Rating':		('decimal', '3,2'),
 			'Read Only':	('varchar', self.VARCHAR_LEN),
 			'Attach':		('text', ''),
 			'Attach Image':	('text', ''),

frappe/database/postgres/database.py

@@ -53,7 +53,7 @@ class PostgresDatabase(Database):
 			'Dynamic Link':	('varchar', self.VARCHAR_LEN),
 			'Password':		('text', ''),
 			'Select':		('varchar', self.VARCHAR_LEN),
-			'Rating':		('smallint', None),
+			'Rating':		('decimal', '3,2'),
 			'Read Only':	('varchar', self.VARCHAR_LEN),
 			'Attach':		('text', ''),
 			'Attach Image':	('text', ''),

frappe/database/query.py

@@ -1,8 +1,10 @@
 import operator
+import re
 from typing import Any, Dict, List, Tuple, Union
 
 import frappe
-from frappe.query_builder import Criterion, Order, Field
+from frappe import _
+from frappe.query_builder import Criterion, Field, Order
 
 
 def like(key: str, value: str) -> frappe.qb:
@@ -224,6 +226,7 @@ class Query:
 		"""
 		conditions = self.get_condition(table, **kwargs)
 		if not filters:
+			conditions = self.add_conditions(conditions, **kwargs)
 			return conditions
 
 		for key in filters:
@@ -245,7 +248,12 @@ class Query:
 		conditions = self.add_conditions(conditions, **kwargs)
 		return conditions
 
-	def build_conditions(self, table: str, filters: Union[Dict[str, Union[str, int]], str, int] = None, **kwargs) -> frappe.qb:
+	def build_conditions(
+		self,
+		table: str,
+		filters: Union[Dict[str, Union[str, int]], str, int] = None,
+		**kwargs
+	) -> frappe.qb:
 		"""Build conditions for sql query
 
 		Args:
@@ -255,13 +263,67 @@ class Query:
 		Returns:
 			frappe.qb: frappe.qb conditions object
 		"""
-		if isinstance(filters, Criterion):
-			return self.criterion_query(table, filters, **kwargs)
-
 		if isinstance(filters, int) or isinstance(filters, str):
 			filters = {"name": str(filters)}
 
-		if isinstance(filters, (list, tuple)):
-			return self.misc_query(table, filters, **kwargs)
+		if isinstance(filters, Criterion):
+			criterion = self.criterion_query(table, filters, **kwargs)
 
-		return self.dict_query(filters=filters, table=table, **kwargs)
+		elif isinstance(filters, (list, tuple)):
+			criterion = self.misc_query(table, filters, **kwargs)
+
+		else:
+			criterion = self.dict_query(filters=filters, table=table, **kwargs)
+
+		return criterion
+
+	def get_sql(
+		self,
+		table: str,
+		fields: Union[List, Tuple],
+		filters: Union[Dict[str, Union[str, int]], str, int] = None,
+		**kwargs
+	):
+		criterion = self.build_conditions(table, filters, **kwargs)
+		if isinstance(fields, (list, tuple)):
+			query = criterion.select(*kwargs.get("field_objects", fields))
+
+		elif isinstance(fields, Criterion):
+			query = criterion.select(fields)
+
+		else:
+			query = criterion.select(fields)
+
+		return query
+
+
+class Permission:
+	@classmethod
+	def check_permissions(cls, query, **kwargs):
+		if not isinstance(query, str):
+			query = query.get_sql()
+
+		doctype = cls.get_tables_from_query(query)
+		if isinstance(doctype, str):
+			doctype = [doctype]
+
+		for dt in doctype:
+			dt = re.sub("tab", "", dt)
+			if not frappe.has_permission(
+				dt,
+				"select",
+				user=kwargs.get("user"),
+				parent_doctype=kwargs.get("parent_doctype"),
+			) and not frappe.has_permission(
+				dt,
+				"read",
+				user=kwargs.get("user"),
+				parent_doctype=kwargs.get("parent_doctype"),
+			):
+				frappe.throw(
+					_("Insufficient Permission for {0}").format(frappe.bold(dt))
+				)
+
+	@staticmethod
+	def get_tables_from_query(query: str):
+		return [table for table in re.findall(r"\w+", query) if table.startswith("tab")]

frappe/deferred_insert.py

@@ -5,7 +5,6 @@ from frappe.utils import cstr
 
 queue_prefix = 'insert_queue_for_'
 
-@frappe.whitelist()
 def deferred_insert(doctype, records):
 	frappe.cache().rpush(queue_prefix + doctype, records)
 

frappe/desk/doctype/form_tour/form_tour.js

@@ -15,10 +15,13 @@ frappe.ui.form.on('Form Tour', {
 
 		frm.add_custom_button(__('Show Tour'), async () => {
 			const issingle = await check_if_single(frm.doc.reference_doctype);
+			const name = await get_first_document(frm.doc.reference_doctype);
 			let route_changed = null;
 
 			if (issingle) {
 				route_changed = frappe.set_route('Form', frm.doc.reference_doctype);
+			} else if (frm.doc.first_document) {
+				route_changed = frappe.set_route('Form', frm.doc.reference_doctype, name);
 			} else {
 				route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new');
 			}
@@ -120,4 +123,15 @@ function get_child_field(child_table, child_name, fieldname) {
 async function check_if_single(doctype) {
 	const { message } = await frappe.db.get_value('DocType', doctype, 'issingle');
 	return message.issingle || 0;
-}
+}
+
+async function get_first_document(doctype) {
+	let docname;
+
+	await frappe.db.get_list(doctype, { order_by: "creation" }).then(res => {
+		if (Array.isArray(res) && res.length)
+			docname = res[0].name;
+	});
+
+	return docname || 'new';
+}

frappe/desk/doctype/form_tour/form_tour.json

@@ -9,8 +9,11 @@
   "title",
   "reference_doctype",
   "module",
+  "column_break_6",
   "is_standard",
   "save_on_complete",
+  "first_document",
+  "include_name_field",
   "section_break_3",
   "steps"
  ],
@@ -62,14 +65,32 @@
    "label": "Module",
    "options": "Module Def",
    "read_only": 1
+  },
+  {
+   "fieldname": "column_break_6",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "fieldname": "first_document",
+   "fieldtype": "Check",
+   "label": "Show First Document Tour"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:!doc.first_document",
+   "fieldname": "include_name_field",
+   "fieldtype": "Check",
+   "label": "Include Name Field"
   }
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-06-06 20:32:54.068774",
+ "modified": "2021-11-24 12:03:45.449311",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Form Tour",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {

frappe/desk/doctype/global_search_settings/global_search_settings.py

@@ -33,7 +33,7 @@ class GlobalSearchSettings(Document):
 
 def get_doctypes_for_global_search():
 	def get_from_db():
-		doctypes = frappe.get_list("Global Search DocType", fields=["document_type"], order_by="idx ASC")
+		doctypes = frappe.get_all("Global Search DocType", fields=["document_type"], order_by="idx ASC")
 		return [d.document_type for d in doctypes] or []
 
 	return frappe.cache().hget("global_search", "search_priorities", get_from_db)

frappe/desk/doctype/kanban_board_column/kanban_board_column.json

@@ -1,155 +1,55 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-10-19 12:26:42.569185", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2016-10-19 12:26:42.569185",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "column_name",
+  "status",
+  "indicator",
+  "order"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Column Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "column_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Column Name"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "Active", 
-   "fieldname": "status", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Status", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Active\nArchived", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "Active",
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Status",
+   "options": "Active\nArchived"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "darkgrey", 
-   "fieldname": "indicator", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Indicator", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "blue\norange\nred\ngreen\ndarkgrey\npurple\nyellow\nlightblue", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "Gray",
+   "fieldname": "indicator",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Indicator",
+   "options": "Blue\nCyan\nGray\nGreen\nLight Blue\nOrange\nPink\nPurple\nRed\nRed\nYellow"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "order", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Order", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "order",
+   "fieldtype": "Code",
+   "label": "Order"
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2017-01-17 15:23:43.520379", 
- "modified_by": "Administrator", 
- "module": "Desk", 
- "name": "Kanban Board Column", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2021-12-14 13:13:38.804259",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "Kanban Board Column",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
 }

frappe/desk/doctype/note/note.css

@@ -1,3 +0,0 @@
-.like-disabled-input{
-    background-color: #fff;
-}

frappe/desk/doctype/onboarding_step/onboarding_step.js

@@ -2,6 +2,17 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Onboarding Step", {
+
+	setup: function(frm) {
+		frm.set_query("form_tour", function() {
+			return {
+				filters: {
+					reference_doctype: frm.doc.reference_document
+				}
+			};
+		});
+	},
+
 	refresh: function(frm) {
 		frappe.boot.developer_mode &&
 			frm.set_intro(

frappe/desk/doctype/onboarding_step/onboarding_step.json

@@ -20,6 +20,7 @@
   "reference_document",
   "show_full_form",
   "show_form_tour",
+  "form_tour",
   "is_single",
   "reference_report",
   "report_reference_doctype",
@@ -206,13 +207,21 @@
    "fieldname": "show_form_tour",
    "fieldtype": "Check",
    "label": "Show Form Tour"
+  },
+  {
+   "depends_on": "show_form_tour",
+   "fieldname": "form_tour",
+   "fieldtype": "Link",
+   "label": "Form Tour",
+   "options": "Form Tour"
   }
  ],
  "links": [],
- "modified": "2020-10-30 14:54:06.646513",
+ "modified": "2021-12-02 10:56:04.448580",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Onboarding Step",
+ "naming_rule": "Set by user",
  "owner": "Administrator",
  "permissions": [
   {

frappe/desk/doctype/route_history/route_history.py

@@ -1,9 +1,13 @@
 # Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
+import json
+
 import frappe
+from frappe.deferred_insert import deferred_insert as _deferred_insert
 from frappe.model.document import Document
 
+
 class RouteHistory(Document):
 	pass
 
@@ -35,3 +39,16 @@ def flush_old_route_records():
 			"modified": ("<=", last_record_to_keep[0].modified),
 			"user": user
 		})
+
+@frappe.whitelist()
+def deferred_insert(routes):
+	routes = [
+		{
+			"user": frappe.session.user,
+			"route": route.get("route"),
+			"creation": route.get("creation"),
+		}
+		for route in frappe.parse_json(routes)
+	]
+
+	_deferred_insert("Route History", json.dumps(routes))

frappe/desk/form/linked_with.py

@@ -2,6 +2,8 @@
 # License: MIT. See LICENSE
 import json
 from collections import defaultdict
+import itertools
+from typing import List
 
 import frappe
 import frappe.desk.form.load
@@ -12,69 +14,296 @@ from frappe.modules import load_doctype_module
 
 
 @frappe.whitelist()
-def get_submitted_linked_docs(doctype, name, docs=None, visited=None):
+def get_submitted_linked_docs(doctype: str, name: str) -> List[tuple]:
+	""" Get all the nested submitted documents those are present in referencing tables (dependent tables).
+
+	:param doctype: Document type
+	:param name: Name of the document
+
+	Usecase:
+	* User should be able to cancel the linked documents along with the one user trying to cancel.
+
+	Case1: If document sd1-n1 (document name n1 from sumittable doctype sd1) is linked to sd2-n2 and sd2-n2 is linked to sd3-n3,
+		Getting submittable linked docs of `sd1-n1`should give both sd2-n2 and sd3-n3.
+	Case2: If document sd1-n1 (document name n1 from sumittable doctype sd1) is linked to d2-n2 and d2-n2 is linked to sd3-n3,
+		Getting submittable linked docs of `sd1-n1`should give None. (because d2-n2 is not a submittable doctype)
+	Case3: If document sd1-n1 (document name n1 from submittable doctype sd1) is linked to d2-n2 & sd2-n2. d2-n2 is linked to sd3-n3.
+		Getting submittable linked docs of `sd1-n1`should give sd2-n2.
+
+	Logic:
+	-----
+	1. We can find linked documents only if we know how the doctypes are related.
+	2. As we need only submittable documents, we can limit doctype relations search to submittable doctypes by
+		finding the relationships(Foreign key references) across submittable doctypes.
+	3. Searching for links is going to be a tree like structure where at every level,
+		you will be finding documents using parent document and parent document links.
 	"""
-	Get all nested submitted linked doctype linkinfo
+	tree = SubmittableDocumentTree(doctype, name)
+	visited_documents = tree.get_all_children()
+	docs = []
 
-	Arguments:
-		doctype (str) - The doctype for which get all linked doctypes
-		name (str) - The docname for which get all linked doctypes
+	for dt, names in visited_documents.items():
+		docs.extend([{'doctype': dt, 'name': name, 'docstatus': 1} for name in names])
 
-	Keyword Arguments:
-		docs (list of dict) - (Optional) Get list of dictionary for linked doctype.
-
-	Returns:
-		dict - Return list of documents and link count
-	"""
-
-	if not docs:
-		docs = []
-
-	if not visited:
-		visited = {}
-
-	if doctype not in visited:
-		visited[doctype] = []
-
-	if name in visited[doctype]:
-		return
-
-	linkinfo = get_linked_doctypes(doctype)
-	linked_docs = get_linked_docs(doctype, name, linkinfo)
-
-	link_count = 0
-	visited[doctype].append(name)
-
-	for link_doctype, link_names in linked_docs.items():
-
-		for link in link_names:
-			if link['name'] == name:
-				continue
-
-			docinfo = link.update({"doctype": link_doctype})
-			validated_doc = validate_linked_doc(docinfo)
-
-			if not validated_doc:
-				continue
-
-			link_count += 1
-
-			links = get_submitted_linked_docs(link_doctype, link.name, docs, visited)
-			if links:
-				docs.append({
-					"doctype": link_doctype,
-					"name": link.name,
-					"docstatus": link.docstatus,
-					"link_count": links.get("count")
-				})
-
-	# sort linked documents by ascending number of links
-	docs.sort(key=lambda doc: doc.get("link_count"))
 	return {
 		"docs": docs,
-		"count": link_count
+		"count": len(docs)
 	}
 
+class SubmittableDocumentTree:
+	def __init__(self, doctype: str, name: str):
+		"""Construct a tree for the submitable linked documents.
+
+		* Node has properties like doctype and docnames. Represented as Node(doctype, docnames).
+		* Nodes are linked by doctype relationships like table, link and dynamic links.
+		* Node is referenced(linked) by many other documents and those are the child nodes.
+
+		NOTE: child document is a property of child node (not same as Frappe child docs of a table field).
+		"""
+		self.root_doctype = doctype
+		self.root_docname = name
+
+		# Documents those are yet to be visited for linked documents.
+		self.to_be_visited_documents = {doctype: [name]}
+		self.visited_documents = defaultdict(list)
+
+		self._submittable_doctypes = None # All submittable doctypes in the system
+		self._references_across_doctypes = None # doctype wise links/references
+
+	def get_all_children(self):
+		"""Get all nodes of a tree except the root node (all the nested submitted
+		documents those are present in referencing tables (dependent tables).
+		"""
+		while self.to_be_visited_documents:
+			next_level_children = defaultdict(list)
+			for parent_dt in list(self.to_be_visited_documents):
+				parent_docs = self.to_be_visited_documents.get(parent_dt)
+				if not parent_docs:
+					del self.to_be_visited_documents[parent_dt]
+					continue
+
+				child_docs = self.get_next_level_children(parent_dt, parent_docs)
+				self.visited_documents[parent_dt].extend(parent_docs)
+				for linked_dt, linked_names in child_docs.items():
+					not_visited_child_docs = set(linked_names) - set(self.visited_documents.get(linked_dt, []))
+					next_level_children[linked_dt].extend(not_visited_child_docs)
+
+			self.to_be_visited_documents = next_level_children
+
+		# Remove root node from visited documents
+		if self.root_docname in self.visited_documents.get(self.root_doctype, []):
+			self.visited_documents[self.root_doctype].remove(self.root_docname)
+
+		return self.visited_documents
+
+	def get_next_level_children(self, parent_dt, parent_names):
+		"""Get immediate children of a Node(parent_dt, parent_names)
+		"""
+		referencing_fields = self.get_doctype_references(parent_dt)
+
+		child_docs = defaultdict(list)
+		for field in referencing_fields:
+			links = get_referencing_documents(parent_dt, parent_names.copy(), field, get_parent_if_child_table_doc=True,
+				parent_filters=[('docstatus', '=', 1)], allowed_parents=self.get_link_sources()) or {}
+			for dt, names in links.items():
+				child_docs[dt].extend(names)
+		return child_docs
+
+	def get_doctype_references(self, doctype):
+		"""Get references for a given document.
+		"""
+		if self._references_across_doctypes is None:
+			get_links_to = self.get_document_sources()
+			limit_link_doctypes = self.get_link_sources()
+			self._references_across_doctypes = get_references_across_doctypes(
+				get_links_to, limit_link_doctypes)
+		return self._references_across_doctypes.get(doctype, [])
+
+	def get_document_sources(self):
+		"""Returns list of doctypes from where we access submittable documents.
+		"""
+		return list(set(self.get_link_sources() + [self.root_doctype]))
+
+	def get_link_sources(self):
+		"""limit doctype links to these doctypes.
+		"""
+		return list(set(self.get_submittable_doctypes()) - set(get_exempted_doctypes() or []))
+
+	def get_submittable_doctypes(self) -> List[str]:
+		"""Returns list of submittable doctypes.
+		"""
+		if not self._submittable_doctypes:
+			self._submittable_doctypes = frappe.db.get_all('DocType', {'is_submittable': 1}, pluck='name')
+		return self._submittable_doctypes
+
+
+def get_child_tables_of_doctypes(doctypes: List[str]=None):
+	"""Returns child tables by doctype.
+	"""
+	filters=[['fieldtype','=', 'Table']]
+	filters_for_docfield = filters
+	filters_for_customfield = filters
+
+	if doctypes:
+		filters_for_docfield = filters + [['parent', 'in', tuple(doctypes)]]
+		filters_for_customfield = filters + [['dt', 'in', tuple(doctypes)]]
+
+	links = frappe.get_all("DocField",
+		fields=["parent", "fieldname", "options as child_table"],
+		filters=filters_for_docfield,
+		as_list=1)
+
+	links+= frappe.get_all("Custom Field",
+		fields=["dt as parent", "fieldname", "options as child_table"],
+		filters=filters_for_customfield,
+		as_list=1)
+
+	child_tables_by_doctype = defaultdict(list)
+	for doctype, fieldname, child_table in links:
+		child_tables_by_doctype[doctype].append(
+			{'doctype': doctype, 'fieldname': fieldname, 'child_table': child_table})
+	return child_tables_by_doctype
+
+
+def get_references_across_doctypes(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None) -> List:
+	"""Find doctype wise foreign key references.
+
+	:param to_doctypes: Get links of these doctypes.
+	:param limit_link_doctypes: limit links to these doctypes.
+
+	* Include child table, link and dynamic link references.
+	"""
+	if limit_link_doctypes:
+		child_tables_by_doctype = get_child_tables_of_doctypes(limit_link_doctypes)
+		all_child_tables = [each['child_table'] for each in itertools.chain(*child_tables_by_doctype.values())]
+		limit_link_doctypes = limit_link_doctypes + all_child_tables
+	else:
+		child_tables_by_doctype = get_child_tables_of_doctypes()
+		all_child_tables = [each['child_table'] for each in itertools.chain(*child_tables_by_doctype.values())]
+
+	references_by_link_fields = get_references_across_doctypes_by_link_field(to_doctypes, limit_link_doctypes)
+	references_by_dlink_fields = get_references_across_doctypes_by_dynamic_link_field(to_doctypes, limit_link_doctypes)
+
+	references = references_by_link_fields.copy()
+	for k, v in references_by_dlink_fields.items():
+		references.setdefault(k, []).extend(v)
+
+	for doctype, links in references.items():
+		for link in links:
+			link['is_child'] = (link['doctype'] in all_child_tables)
+	return references
+
+
+def get_references_across_doctypes_by_link_field(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None):
+	"""Find doctype wise foreign key references based on link fields.
+
+	:param to_doctypes: Get links to these doctypes.
+	:param limit_link_doctypes: limit links to these doctypes.
+	"""
+	filters=[['fieldtype','=', 'Link']]
+
+	if to_doctypes:
+		filters += [['options', 'in', tuple(to_doctypes)]]
+
+	filters_for_docfield = filters[:]
+	filters_for_customfield = filters[:]
+
+	if limit_link_doctypes:
+		filters_for_docfield += [['parent', 'in', tuple(limit_link_doctypes)]]
+		filters_for_customfield += [['dt', 'in', tuple(limit_link_doctypes)]]
+
+	links = frappe.get_all("DocField",
+		fields=["parent", "fieldname", "options as linked_to"],
+		filters=filters_for_docfield,
+		as_list=1)
+
+	links+= frappe.get_all("Custom Field",
+		fields=["dt as parent", "fieldname", "options as linked_to"],
+		filters=filters_for_customfield,
+		as_list=1)
+
+	links_by_doctype = defaultdict(list)
+	for doctype, fieldname, linked_to in links:
+		links_by_doctype[linked_to].append({'doctype': doctype, 'fieldname': fieldname})
+	return links_by_doctype
+
+
+def get_references_across_doctypes_by_dynamic_link_field(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None):
+	"""Find doctype wise foreign key references based on dynamic link fields.
+
+	:param to_doctypes: Get links to these doctypes.
+	:param limit_link_doctypes: limit links to these doctypes.
+	"""
+
+	filters=[['fieldtype','=', 'Dynamic Link']]
+
+	filters_for_docfield = filters[:]
+	filters_for_customfield = filters[:]
+
+	if limit_link_doctypes:
+		filters_for_docfield += [['parent', 'in', tuple(limit_link_doctypes)]]
+		filters_for_customfield += [['dt', 'in', tuple(limit_link_doctypes)]]
+
+	# find dynamic links of parents
+	links = frappe.get_all("DocField",
+		fields=["parent as doctype", "fieldname", "options as doctype_fieldname"],
+		filters=filters_for_docfield,
+		as_list=1)
+
+	links += frappe.get_all("Custom Field",
+		fields=["dt as doctype", "fieldname", "options as doctype_fieldname"],
+		filters=filters_for_customfield,
+		as_list=1)
+
+	links_by_doctype = defaultdict(list)
+	for doctype, fieldname, doctype_fieldname in links:
+		try:
+			filters = [[doctype_fieldname, 'in', to_doctypes]] if to_doctypes else []
+			for linked_to in frappe.db.get_all(doctype, pluck=doctype_fieldname, filters = filters, distinct=1):
+				if linked_to:
+					links_by_doctype[linked_to].append({'doctype': doctype, 'fieldname': fieldname, 'doctype_fieldname': doctype_fieldname})
+		except frappe.db.ProgrammingError:
+			# TODO: FIXME
+			continue
+	return links_by_doctype
+
+def get_referencing_documents(reference_doctype: str, reference_names: List[str],
+		link_info: dict, get_parent_if_child_table_doc: bool=True,
+		parent_filters: List[list]=None, child_filters=None, allowed_parents=None):
+	"""Get linked documents based on link_info.
+
+	:param reference_doctype: reference doctype to find links
+	:param reference_names: reference document names to find links for
+	:param link_info: linking details to get the linked documents
+		Ex: {'doctype': 'Purchase Invoice Advance', 'fieldname': 'reference_name',
+			'doctype_fieldname': 'reference_type', 'is_child': True}
+	:param get_parent_if_child_table_doc: Get parent record incase linked document is a child table record.
+	:param parent_filters: filters to apply on if not a child table.
+	:param child_filters: apply filters if it is a child table.
+	:param allowed_parents: list of parents allowed in case of get_parent_if_child_table_doc
+		is enabled.
+	"""
+	from_table = link_info['doctype']
+	filters = [[link_info['fieldname'], 'in', tuple(reference_names)]]
+	if link_info.get('doctype_fieldname'):
+		filters.append([link_info['doctype_fieldname'], '=', reference_doctype])
+
+	if not link_info.get('is_child'):
+		filters.extend(parent_filters or [])
+		return {from_table: frappe.db.get_all(from_table, filters, pluck='name')}
+
+
+	filters.extend(child_filters or [])
+	res = frappe.db.get_all(from_table, filters = filters, fields = ['name', 'parenttype', 'parent'])
+	documents = defaultdict(list)
+
+	for parent, rows in itertools.groupby(res, key = lambda row: row['parenttype']):
+		if allowed_parents and parent not in allowed_parents:
+			continue
+		filters = (parent_filters or []) + [['name', 'in', tuple([row.parent for row in rows])]]
+		documents[parent].extend(frappe.db.get_all(parent, filters=filters, pluck='name') or [])
+	return documents
+
 
 @frappe.whitelist()
 def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=None):
@@ -109,7 +338,6 @@ def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=None):
 	Returns:
 		bool: True if linked document passes all validations, else False
 	"""
-
 	#ignore doctype to cancel
 	if docinfo.get("doctype") in (ignore_doctypes_on_cancel_all or []):
 		return False
@@ -132,7 +360,6 @@ def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=None):
 
 def get_exempted_doctypes():
 	""" Get list of doctypes exempted from being auto-cancelled """
-
 	auto_cancel_exempt_doctypes = []
 	for doctypes in frappe.get_hooks('auto_cancel_exempted_doctypes'):
 		auto_cancel_exempt_doctypes.append(doctypes)
@@ -183,11 +410,11 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 
 			try:
 				if link.get("filters"):
-					ret = frappe.get_list(doctype=dt, fields=fields, filters=link.get("filters"))
+					ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters"))
 
 				elif link.get("get_parent"):
 					if me and me.parent and me.parenttype == dt:
-						ret = frappe.get_list(doctype=dt, fields=fields,
+						ret = frappe.get_all(doctype=dt, fields=fields,
 							filters=[[dt, "name", '=', me.parent]])
 					else:
 						ret = None
@@ -199,7 +426,7 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 					if link.get("doctype_fieldname"):
 						filters.append([link.get('child_doctype'), link.get("doctype_fieldname"), "=", doctype])
 
-					ret = frappe.get_list(doctype=dt, fields=fields, filters=filters, or_filters=or_filters, distinct=True)
+					ret = frappe.get_all(doctype=dt, fields=fields, filters=filters, or_filters=or_filters, distinct=True)
 
 				else:
 					link_fieldnames = link.get("fieldname")
@@ -210,7 +437,7 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 						# dynamic link
 						if link.get("doctype_fieldname"):
 							filters.append([dt, link.get("doctype_fieldname"), "=", doctype])
-						ret = frappe.get_list(doctype=dt, fields=fields, filters=filters, or_filters=or_filters)
+						ret = frappe.get_all(doctype=dt, fields=fields, filters=filters, or_filters=or_filters)
 
 					else:
 						ret = None

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -197,6 +197,8 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 			callback: (r) => {
 				if (r.message.status === 'ok') {
 					this.post_setup_success();
+				} else if (r.message.status === 'registered') {
+					this.update_setup_message(__("starting the setup..."));
 				} else if (r.message.fail !== undefined) {
 					this.abort_setup(r.message.fail);
 				}
@@ -238,6 +240,9 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 			if (data.fail_msg) {
 				this.abort_setup(data.fail_msg);
 			}
+			if (data.status === 'ok') {
+				this.post_setup_success();
+			}
 		})
 	}
 

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -54,9 +54,17 @@ def setup_complete(args):
 		return {'status': 'ok'}
 
 	args = parse_args(args)
-
 	stages = get_setup_stages(args)
+	is_background_task = frappe.conf.get('trigger_site_setup_in_background')
 
+	if is_background_task:
+		process_setup_stages.enqueue(stages=stages, user_input=args, is_background_task=True)
+		return {'status': 'registered'}
+	else:
+		return process_setup_stages(stages, args)
+
+@frappe.task()
+def process_setup_stages(stages, user_input, is_background_task=False):
 	try:
 		frappe.flags.in_setup_wizard = True
 		current_task = None
@@ -68,11 +76,16 @@ def setup_complete(args):
 				current_task = task
 				task.get('fn')(task.get('args'))
 	except Exception:
-		handle_setup_exception(args)
-		return {'status': 'fail', 'fail': current_task.get('fail_msg')}
+		handle_setup_exception(user_input)
+		if not is_background_task:
+			return {'status': 'fail', 'fail': current_task.get('fail_msg')}
+		frappe.publish_realtime('setup_task',
+			{'status': 'fail', "fail_msg": current_task.get('fail_msg')}, user=frappe.session.user)
 	else:
-		run_setup_success(args)
-		return {'status': 'ok'}
+		run_setup_success(user_input)
+		if not is_background_task:
+			return {'status': 'ok'}
+		frappe.publish_realtime('setup_task', {"status": 'ok'}, user=frappe.session.user)
 	finally:
 		frappe.flags.in_setup_wizard = False
 

frappe/desk/page/user_profile/user_profile_controller.js

@@ -17,21 +17,15 @@ class UserProfile {
 	show() {
 		let route = frappe.get_route();
 		this.user_id = route[1] || frappe.session.user;
-
-		//validate if user
-		if (route.length > 1) {
-			frappe.dom.freeze(__('Loading user profile') + '...');
-			frappe.db.exists('User', this.user_id).then(exists => {
-				frappe.dom.unfreeze();
-				if (exists) {
-					this.make_user_profile();
-				} else {
-					frappe.msgprint(__('User does not exist'));
-				}
-			});
-		} else {
-			frappe.set_route('user-profile', frappe.session.user);
-		}
+		frappe.dom.freeze(__('Loading user profile') + '...');
+		frappe.db.exists('User', this.user_id).then(exists => {
+			frappe.dom.unfreeze();
+			if (exists) {
+				this.make_user_profile();
+			} else {
+				frappe.msgprint(__('User does not exist'));
+			}
+		});
 	}
 
 	make_user_profile() {
@@ -74,8 +68,7 @@ class UserProfile {
 			primary_action_label: __('Go'),
 			primary_action: ({ user }) => {
 				dialog.hide();
-				this.user_id = user;
-				this.make_user_profile();
+				frappe.set_route('user-profile', user);
 			}
 		});
 		dialog.show();

frappe/desk/page/user_profile/user_profile_sidebar.html

@@ -51,10 +51,10 @@
 			<p><a class="edit-profile-link">{%=__("Edit Profile") %}</a></p>
 			<p><a class="user-settings-link">{%=__("User Settings") %}</a></p>
 			<p>
-				<a class="leaderboard-link" href="#leaderboard/User"
+				<a class="leaderboard-link" href="/app/leaderboard/User"
 					>{%=__("Leaderboard") %}</a
 				>
 			</p>
 		</div>
 	</div>
-</div>
+</div>

frappe/email/doctype/email_account/email_account.js

@@ -109,6 +109,15 @@ frappe.ui.form.on("Email Account", {
 	onload: function(frm) {
 		frm.set_df_property("append_to", "only_select", true);
 		frm.set_query("append_to", "frappe.email.doctype.email_account.email_account.get_append_to");
+		frm.set_query("append_to", "imap_folder", function() {
+			return {
+				query: "frappe.email.doctype.email_account.email_account.get_append_to"
+			};
+		});
+		if (frm.doc.__islocal) {
+			frm.add_child("imap_folder", {"folder_name": "INBOX"});
+			frm.refresh_field("imap_folder");
+		}
 	},
 
 	refresh: function(frm) {
@@ -117,7 +126,7 @@ frappe.ui.form.on("Email Account", {
 		frm.events.notify_if_unreplied(frm);
 		frm.events.show_gmail_message_for_less_secure_apps(frm);
 
-		if(frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) {
+		if (frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) {
 			delete frappe.route_flags.delete_user_from_locals;
 			delete locals['User'][frappe.route_flags.linked_user];
 		}
@@ -125,7 +134,7 @@ frappe.ui.form.on("Email Account", {
 
 	show_gmail_message_for_less_secure_apps: function(frm) {
 		frm.dashboard.clear_headline();
-		if(frm.doc.service==="GMail") {
+		if (frm.doc.service==="GMail") {
 			frm.dashboard.set_headline_alert('Gmail will only work if you allow access for less secure \
 				apps in Gmail settings. <a target="_blank" \
 				href="https://support.google.com/accounts/answer/6010255?hl=en">Read this for details</a>');
@@ -137,8 +146,8 @@ frappe.ui.form.on("Email Account", {
 		frm.events.update_domain(frm);
 	},
 
-	update_domain: function(frm){
-		if (!frm.doc.email_id && !frm.doc.service){
+	update_domain: function(frm) {
+		if (!frm.doc.email_id && !frm.doc.service) {
 			return;
 		}
 
@@ -148,7 +157,7 @@ frappe.ui.form.on("Email Account", {
 			args: {
 				"email_id": frm.doc.email_id
 			},
-			callback: function (r) {
+			callback: function(r) {
 				if (r.message) {
 					frm.events.set_domain_fields(frm, r.message);
 				}
@@ -157,7 +166,7 @@ frappe.ui.form.on("Email Account", {
 	},
 
 	set_domain_fields: function(frm, args) {
-		if(!args){
+		if (!args) {
 			args = frappe.route_flags.set_domain_values? frappe.route_options: {};
 		}
 
@@ -172,10 +181,8 @@ frappe.ui.form.on("Email Account", {
 	email_sync_option: function(frm) {
 		// confirm if the ALL sync option is selected
 
-		if(frm.doc.email_sync_option == "ALL"){
-			var msg = __("You are selecting Sync Option as ALL, It will resync all \
-				read as well as unread message from server. This may also cause the duplication\
-				of Communication (emails).");
+		if (frm.doc.email_sync_option == "ALL") {
+			var msg = __("You are selecting Sync Option as ALL, It will resync all read as well as unread message from server. This may also cause the duplication of Communication (emails).");
 			frappe.confirm(msg, null, function() {
 				frm.set_value("email_sync_option", "UNSEEN");
 			});
@@ -184,8 +191,7 @@ frappe.ui.form.on("Email Account", {
 
 	warn_autoreply_on_incoming: function(frm) {
 		if (frm.doc.enable_incoming && frm.doc.enable_auto_reply && frm.doc.__islocal) {
-			var msg = __("Enabling auto reply on an incoming email account will send automated replies \
-				to all the synchronized emails. Do you wish to continue?");
+			var msg = __("Enabling auto reply on an incoming email account will send automated replies to all the synchronized emails. Do you wish to continue?");
 			frappe.confirm(msg, null, function() {
 				frm.set_value("enable_auto_reply", 0);
 				frappe.show_alert({message: __("Disabled Auto Reply"), indicator: "blue"});

frappe/email/doctype/email_account/email_account.json

@@ -31,6 +31,8 @@
   "attachment_limit",
   "email_sync_option",
   "initial_sync_count",
+  "section_break_25",
+  "imap_folder",
   "section_break_12",
   "append_emails_to_sent_folder",
   "append_to",
@@ -204,7 +206,7 @@
    "label": "Attachment Limit (MB)"
   },
   {
-   "depends_on": "enable_incoming",
+   "depends_on": "eval: doc.enable_incoming && !doc.use_imap",
    "description": "Append as communication against this DocType (must have fields, \"Status\", \"Subject\")",
    "fieldname": "append_to",
    "fieldtype": "Link",
@@ -562,15 +564,28 @@
    "fieldname": "account_section",
    "fieldtype": "Section Break",
    "label": "Account"
+  },
+  {
+   "depends_on": "eval: doc.use_imap && doc.enable_incoming",
+   "fieldname": "imap_folder",
+   "fieldtype": "Table",
+   "label": "IMAP Folder",
+   "options": "IMAP Folder"
+  },
+  {
+   "fieldname": "section_break_25",
+   "fieldtype": "Section Break",
+   "label": "IMAP Details"
   }
  ],
  "icon": "fa fa-inbox",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-09-21 16:44:25.728637",
+ "modified": "2021-11-30 09:03:25.728637",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Account",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {

frappe/email/doctype/email_account/email_account.py

@@ -67,6 +67,10 @@ class EmailAccount(Document):
 		else:
 			self.login_id = None
 
+		# validate the imap settings
+		if self.enable_incoming and self.use_imap and len(self.imap_folder) <= 0:
+			frappe.throw(_("You need to set one IMAP folder for {0}").format(frappe.bold(self.email_id)))
+
 		duplicate_email_account = frappe.get_all("Email Account", filters={
 			"email_id": self.email_id,
 			"name": ("!=", self.name)
@@ -100,10 +104,11 @@ class EmailAccount(Document):
 			for e in self.get_unreplied_notification_emails():
 				validate_email_address(e, True)
 
-		if self.enable_incoming and self.append_to:
-			valid_doctypes = [d[0] for d in get_append_to()]
-			if self.append_to not in valid_doctypes:
-				frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
+		for folder in self.imap_folder:
+			if self.enable_incoming and folder.append_to:
+				valid_doctypes = [d[0] for d in get_append_to()]
+				if folder.append_to not in valid_doctypes:
+					frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
 
 	def validate_smtp_conn(self):
 		if not self.smtp_server:
@@ -177,13 +182,13 @@ class EmailAccount(Document):
 			return None
 
 		args = frappe._dict({
+			"email_account_name": self.email_account_name,
 			"email_account": self.name,
 			"host": self.email_server,
 			"use_ssl": self.use_ssl,
 			"username": getattr(self, "login_id", None) or self.email_id,
 			"use_imap": self.use_imap,
 			"email_sync_rule": email_sync_rule,
-			"uid_validity": self.uidvalidity,
 			"incoming_port": get_port(self),
 			"initial_sync_count": self.initial_sync_count or 100
 		})
@@ -457,6 +462,14 @@ class EmailAccount(Document):
 		"""retrive and return inbound mails.
 
 		"""
+		mails = []
+
+		def process_mail(messages):
+			for index, message in enumerate(messages.get("latest_messages", [])):
+				uid = messages['uid_list'][index] if messages.get('uid_list') else None
+				seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0
+				mails.append(InboundMail(message, self, uid, seen_status))
+
 		if frappe.local.flags.in_test:
 			return [InboundMail(msg, self) for msg in test_mails or []]
 
@@ -466,17 +479,23 @@ class EmailAccount(Document):
 		email_sync_rule = self.build_email_sync_rule()
 		try:
 			email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule)
-			messages = email_server.get_messages() or {}
+			if self.use_imap:
+				# process all given imap folder
+				for folder in self.imap_folder:
+					email_server.select_imap_folder(folder.folder_name)
+					email_server.settings['uid_validity'] = folder.uidvalidity
+					messages = email_server.get_messages(folder=folder.folder_name) or {}
+					process_mail(messages)
+			else:
+				# process the pop3 account
+				messages = email_server.get_messages() or {}
+				process_mail(messages)
+			# close connection to mailserver
+			email_server.logout()
 		except Exception:
 			frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
 			return []
 
-		mails = []
-		for index, message in enumerate(messages.get("latest_messages", [])):
-			uid = messages['uid_list'][index] if messages.get('uid_list') else None
-			seen_status = 1 if messages.get('seen_status', {}).get(uid)=='SEEN' else 0
-			mails.append(InboundMail(message, self, uid, seen_status))
-
 		return mails
 
 	def handle_bad_emails(self, uid, raw, reason):
@@ -530,7 +549,11 @@ class EmailAccount(Document):
 
 	def on_trash(self):
 		"""Clear communications where email account is linked"""
-		frappe.db.sql("update `tabCommunication` set email_account='' where email_account=%s", self.name)
+		Communication = frappe.qb.DocType("Communication")
+		frappe.qb.update(Communication) \
+			.set(Communication.email_account, "") \
+			.where(Communication.email_account == self.name).run()
+
 		remove_user_email_inbox(email_account=self.name)
 
 	def after_rename(self, old, new, merge=False):
@@ -547,23 +570,26 @@ class EmailAccount(Document):
 		else:
 			return self.email_sync_option or "UNSEEN"
 
-	def mark_emails_as_read_unread(self):
+	def mark_emails_as_read_unread(self, email_server=None, folder_name="INBOX"):
 		""" mark Email Flag Queue of self.email_account mails as read"""
-
 		if not self.use_imap:
 			return
 
-		flags = frappe.db.sql("""select name, communication, uid, action from
-			`tabEmail Flag Queue` where is_completed=0 and email_account={email_account}
-			""".format(email_account=frappe.db.escape(self.name)), as_dict=True)
+		EmailFlagQ = frappe.qb.DocType("Email Flag Queue")
+		flags = (
+			frappe.qb.from_(EmailFlagQ)
+			.select(EmailFlagQ.name, EmailFlagQ.communication, EmailFlagQ.uid, EmailFlagQ.action)
+			.where(EmailFlagQ.is_completed == 0)
+			.where(EmailFlagQ.email_account == frappe.db.escape(self.name))
+		).run(as_dict=True)
 
 		uid_list = { flag.get("uid", None): flag.get("action", "Read") for flag in flags }
 		if flags and uid_list:
-			email_server = self.get_incoming_server()
+			if not email_server:
+				email_server = self.get_incoming_server()
 			if not email_server:
 				return
-
-			email_server.update_flag(uid_list=uid_list)
+			email_server.update_flag(folder_name, uid_list=uid_list)
 
 			# mark communication as read
 			docnames =  ",".join("'%s'"%flag.get("communication") for flag in flags \
@@ -576,16 +602,20 @@ class EmailAccount(Document):
 			self.set_communication_seen_status(docnames, seen=0)
 
 			docnames = ",".join([ "'%s'"%flag.get("name") for flag in flags ])
-			frappe.db.sql(""" update `tabEmail Flag Queue` set is_completed=1
-				where name in ({docnames})""".format(docnames=docnames))
+
+			EmailFlagQueue = frappe.qb.DocType("Email Flag Queue")
+			frappe.qb.update(EmailFlagQueue) \
+				.set(EmailFlagQueue.is_completed, 1) \
+				.where(EmailFlagQueue.name.isin(docnames)).run()
 
 	def set_communication_seen_status(self, docnames, seen=0):
 		""" mark Email Flag Queue of self.email_account mails as read"""
 		if not docnames:
 			return
-
-		frappe.db.sql(""" update `tabCommunication` set seen={seen}
-			where name in ({docnames})""".format(docnames=docnames, seen=seen))
+		Communication = frappe.qb.from_("Communication")
+		frappe.qb.update(Communication) \
+			.set(Communication.seen == seen) \
+			.where(Communication.name.isin(docnames)).run()
 
 	def check_automatic_linking_email_account(self):
 		if self.enable_automatic_linking:
@@ -651,15 +681,19 @@ def test_internet(host="8.8.8.8", port=53, timeout=3):
 def notify_unreplied():
 	"""Sends email notifications if there are unreplied Communications
 		and `notify_if_unreplied` is set as true."""
-
 	for email_account in frappe.get_all("Email Account", "name", filters={"enable_incoming": 1, "notify_if_unreplied": 1}):
 		email_account = frappe.get_doc("Email Account", email_account.name)
-		if email_account.append_to:
 
+		if email_account.use_imap:
+			append_to = [folder.get("append_to") for folder in email_account.imap_folder]
+		else:
+			append_to = email_account.append_to
+
+		if append_to:
 			# get open communications younger than x mins, for given doctype
 			for comm in frappe.get_all("Communication", "name", filters=[
 					{"sent_or_received": "Received"},
-					{"reference_doctype": email_account.append_to},
+					{"reference_doctype": ("in", append_to)},
 					{"unread_notification_sent": 0},
 					{"email_account":email_account.name},
 					{"creation": ("<", datetime.now() - timedelta(seconds = (email_account.unreplied_for_mins or 30) * 60))},
@@ -702,9 +736,6 @@ def pull_from_email_account(email_account):
 	email_account = frappe.get_doc("Email Account", email_account)
 	email_account.receive()
 
-	# mark Email Flag Queue mail as read
-	email_account.mark_emails_as_read_unread()
-
 def get_max_email_uid(email_account):
 	# get maximum uid of emails
 	max_uid = 1
@@ -761,12 +792,12 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou
 			update_user_email_settings = True
 
 	if update_user_email_settings:
-		frappe.db.sql("""UPDATE `tabUser Email` SET awaiting_password = %(awaiting_password)s,
-			enable_outgoing = %(enable_outgoing)s WHERE email_account = %(email_account)s""", {
-				"email_account": email_account,
-				"enable_outgoing": enable_outgoing,
-				"awaiting_password": awaiting_password or 0
-			})
+		UserEmail = frappe.qb.DocType("User Email")
+		frappe.qb.update(UserEmail) \
+			.set(UserEmail.awaiting_password, (awaiting_password or 0)) \
+			.set(UserEmail.enable_outgoing, enable_outgoing) \
+			.where(UserEmail.email_account == email_account).run()
+
 	else:
 		users = " and ".join([frappe.bold(user.get("name")) for user in user_names])
 		frappe.msgprint(_("Enabled email inbox for user {0}").format(users))
@@ -800,4 +831,4 @@ def set_email_password(email_account, user, password):
 			frappe.db.rollback()
 			return False
 
-	return True
+	return True

frappe/email/doctype/email_account/test_email_account.py

@@ -25,6 +25,7 @@ class TestEmailAccount(unittest.TestCase):
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
 		email_account.db_set("enable_incoming", 1)
 		email_account.db_set("enable_auto_reply", 1)
+		email_account.db_set("use_imap", 1)
 
 	@classmethod
 	def tearDownClass(cls):
@@ -229,6 +230,22 @@ class TestEmailAccount(unittest.TestCase):
 		email_account.handle_bad_emails(uid=-1, raw=mail_content, reason="Testing")
 		self.assertTrue(frappe.db.get_value("Unhandled Email", {'message_id': message_id}))
 
+	def test_imap_folder(self):
+		# assert tests if imap_folder >= 1 and imap is checked
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+
+		self.assertTrue(email_account.use_imap)
+		self.assertTrue(email_account.enable_incoming)
+		self.assertTrue(len(email_account.imap_folder) > 0)
+
+	def test_imap_folder_missing(self):
+		# Test the Exception in validate() that verifies the imap_folder list
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.imap_folder = []
+
+		with self.assertRaises(Exception):
+			email_account.validate()
+
 class TestInboundMail(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):

frappe/email/doctype/email_account/test_records.json

@@ -4,7 +4,6 @@
 		"is_global": 1,
 		"doctype": "Email Account",
 		"domain":"example.com",
-		"append_to": "ToDo",
 		"email_account_name": "_Test Email Account 1",
 		"enable_outgoing": 1,
 		"smtp_server": "test.example.com",
@@ -20,6 +19,8 @@
 		"send_notification_to": "test_unreplied@example.com",
 		"pop3_server": "pop.test.example.com",
 		"no_remaining":"0",
+		"append_to": "ToDo",
+		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}],
 		"track_email_status": 1
 	},
 	{

frappe/email/doctype/email_flag_queue/email_flag_queue.json

@@ -1,213 +1,67 @@
 {
- "allow_copy": 1, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-04-20 15:29:39.785172", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 0, 
+ "actions": [],
+ "allow_copy": 1,
+ "creation": "2016-04-20 15:29:39.785172",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "is_completed",
+  "communication",
+  "action",
+  "email_account",
+  "uid"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "is_completed", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Is Completed", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "is_completed",
+   "fieldtype": "Check",
+   "label": "Is Completed",
+   "read_only": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "communication", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Communication", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "communication",
+   "fieldtype": "Data",
+   "label": "Communication"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "action", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Action", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Read\nUnread", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "action",
+   "fieldtype": "Select",
+   "label": "Action",
+   "options": "Read\nUnread"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "email_account", 
-   "fieldtype": "Data", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Email Account", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "email_account",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Email Account"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "uid", 
-   "fieldtype": "Data", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "UID", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "uid",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "UID"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-09-20 15:27:12.142079", 
- "modified_by": "Administrator", 
- "module": "Email", 
- "name": "Email Flag Queue", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "in_create": 1,
+ "links": [],
+ "modified": "2021-11-30 09:51:34.489932",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Email Flag Queue",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 0
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC"
 }

frappe/email/doctype/email_queue/email_queue.py

@@ -18,7 +18,7 @@ from frappe import _, safe_encode, task
 from frappe.model.document import Document
 from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message
 from frappe.email.email_body import add_attachment, get_formatted_html, get_email
-from frappe.utils import cint, split_emails, add_days, nowdate, cstr
+from frappe.utils import cint, split_emails, add_days, nowdate, cstr, get_hook_method
 from frappe.email.doctype.email_account.email_account import EmailAccount
 
 
@@ -121,9 +121,13 @@ class EmailQueue(Document):
 					continue
 
 				message = ctx.build_message(recipient.recipient)
-				if not frappe.flags.in_test:
-					ctx.smtp_session.sendmail(from_addr=self.sender, to_addrs=recipient.recipient, msg=message)
-				ctx.add_to_sent_list(recipient)
+				method = get_hook_method('override_email_send')
+				if method:
+					method(self, self.sender, recipient.recipient, message)
+				else:
+					if not frappe.flags.in_test:
+						ctx.smtp_session.sendmail(from_addr=self.sender, to_addrs=recipient.recipient, msg=message)
+					ctx.add_to_sent_list(recipient)
 
 			if frappe.flags.in_test:
 				frappe.flags.sent_mail = message
@@ -283,9 +287,14 @@ class SendMailContext:
 			if attachment.get('fcontent'):
 				continue
 
-			fid = attachment.get("fid")
-			if fid:
-				_file = frappe.get_doc("File", fid)
+			file_filters = {}
+			if attachment.get('fid'):
+				file_filters['name'] = attachment.get('fid')
+			elif attachment.get('file_url'):
+				file_filters['file_url'] = attachment.get('file_url')
+
+			if file_filters:
+				_file = frappe.get_doc("File", file_filters)
 				fcontent = _file.get_content()
 				attachment.update({
 					'fname': _file.file_name,
@@ -293,6 +302,7 @@ class SendMailContext:
 					'parent': message_obj
 				})
 				attachment.pop("fid", None)
+				attachment.pop("file_url", None)
 				add_attachment(**attachment)
 
 			elif attachment.get("print_format_attachment") == 1:
@@ -503,7 +513,7 @@ class QueueBuilder:
 		if self._attachments:
 			# store attachments with fid or print format details, to be attached on-demand later
 			for att in self._attachments:
-				if att.get('fid'):
+				if att.get('fid') or att.get('file_url'):
 					attachments.append(att)
 				elif att.get("print_format_attachment") == 1:
 					if not att.get('lang', None):

frappe/email/doctype/imap_folder/imap_folder.json

@@ -0,0 +1,53 @@
+{
+ "actions": [],
+ "creation": "2021-09-21 11:38:13.521979",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "folder_name",
+  "append_to",
+  "uidvalidity",
+  "uidnext"
+ ],
+ "fields": [
+  {
+   "fieldname": "folder_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Folder Name",
+   "reqd": 1
+  },
+  {
+   "fieldname": "append_to",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Append To",
+   "options": "DocType"
+  },
+  {
+   "fieldname": "uidvalidity",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "UIDVALIDITY"
+  },
+  {
+   "fieldname": "uidnext",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "UIDNEXT"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-09-21 11:53:00.811236",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "IMAP Folder",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/email/doctype/imap_folder/imap_folder.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class IMAPFolder(Document):
+	pass

frappe/email/doctype/newsletter/newsletter.js

@@ -4,69 +4,137 @@
 frappe.ui.form.on('Newsletter', {
 	refresh(frm) {
 		let doc = frm.doc;
-		if (!doc.__islocal && !cint(doc.email_sent) && !doc.__unsaved
-				&& in_list(frappe.boot.user.can_write, doc.doctype)) {
-			frm.add_custom_button(__('Send Now'), function() {
-				frappe.confirm(__("Do you really want to send this email newsletter?"), function() {
-					frm.call('send_emails').then(() => {
-						frm.refresh();
-					});
+		let can_write = in_list(frappe.boot.user.can_write, doc.doctype);
+		if (!frm.is_new() && !frm.is_dirty() && !doc.email_sent && can_write) {
+			frm.add_custom_button(__('Send a test email'), () => {
+				frm.events.send_test_email(frm);
+			}, __('Preview'));
+
+			frm.add_custom_button(__('Check broken links'), () => {
+				frm.dashboard.set_headline(__('Checking broken links...'));
+				frm.call('find_broken_links').then(r => {
+					frm.dashboard.set_headline('');
+					let links = r.message;
+					if (links && links.length) {
+						let html = '<ul>' + links.map(link => `<li>${link}</li>`).join('') + '</ul>';
+						frm.dashboard.set_headline(__("Following links are broken in the email content: {0}", [html]));
+					} else {
+						frm.dashboard.set_headline(__("No broken links found in the email content"));
+						setTimeout(() => {
+							frm.dashboard.set_headline('');
+						}, 3000);
+					}
 				});
-			}, "fa fa-play", "btn-success");
+			}, __('Preview'));
+
+			frm.add_custom_button(__('Send now'), () => {
+				if (frm.doc.schedule_send) {
+					frappe.confirm(__("This newsletter was scheduled to send on a later date. Are you sure you want to send it now?"), function () {
+						frm.call('send_emails').then(() => frm.refresh());
+					});
+					return;
+				}
+				frappe.confirm(__("Are you sure you want to send this newsletter now?"), function () {
+					frm.call('send_emails').then(() => frm.refresh());
+				});
+			}, __('Send'));
+
+			frm.add_custom_button(__('Schedule sending'), () => {
+				frm.events.schedule_send_dialog(frm);
+			}, __('Send'));
 		}
 
 		frm.events.setup_dashboard(frm);
+		frm.events.setup_sending_status(frm);
 
-		if (doc.__islocal && !doc.send_from) {
+		if (frm.is_new() && !doc.sender_email) {
 			let { fullname, email } = frappe.user_info(doc.owner);
-			frm.set_value('send_from', `${fullname} <${email}>`);
+			frm.set_value('sender_email', email);
+			frm.set_value('sender_name', fullname);
 		}
+
+		frm.trigger('update_schedule_message');
 	},
 
-	onload_post_render(frm) {
-		frm.trigger('setup_schedule_send');
-	},
-
-	setup_schedule_send(frm) {
-		let today = new Date();
-
-		// setting datepicker options to set min date & min time
-		today.setHours(today.getHours() + 1 );
-		frm.get_field('schedule_send').$input.datepicker({
-			maxMinutes: 0,
-			minDate: today,
-			timeFormat: 'hh:00:00',
-			onSelect: function (fd, d, picker) {
-				if (!d) return;
-				var date = d.toDateString();
-				if (date === today.toDateString()) {
-					picker.update({
-						minHours: (today.getHours() + 1)
-					});
-				} else {
-					picker.update({
-						minHours: 0
-					});
-				}
-				frm.get_field('schedule_send').$input.trigger('change');
+	schedule_send_dialog(frm) {
+		let hours = frappe.utils.range(24);
+		let time_slots = hours.map(hour => {
+			return `${(hour + '').padStart(2, '0')}:00`;
+		});
+		let d = new frappe.ui.Dialog({
+			title: __('Schedule Newsletter'),
+			fields: [
+				{
+					label: __('Date'),
+					fieldname: 'date',
+					fieldtype: 'Date',
+					options: {
+						minDate: new Date()
+					}
+				},
+				{
+					label: __('Time'),
+					fieldname: 'time',
+					fieldtype: 'Select',
+					options: time_slots,
+				},
+			],
+			primary_action_label: __('Schedule'),
+			primary_action({ date, time }) {
+				frm.set_value('schedule_sending', 1);
+				frm.set_value('schedule_send', `${date} ${time}:00`);
+				d.hide();
+				frm.save();
+			},
+			secondary_action_label: __('Cancel Scheduling'),
+			secondary_action() {
+				frm.set_value('schedule_sending', 0);
+				frm.set_value('schedule_send', '');
+				d.hide();
+				frm.save();
 			}
 		});
+		if (frm.doc.schedule_sending) {
+			let parts = frm.doc.schedule_send.split(' ');
+			if (parts.length === 2) {
+				let [date, time] = parts;
+				d.set_value('date', date);
+				d.set_value('time', time.slice(0, 5));
+			}
+		}
+		d.show();
+	},
 
-
-		const $tp = frm.get_field('schedule_send').datepicker.timepicker;
-		$tp.$minutes.parent().css('display', 'none');
-		$tp.$minutesText.css('display', 'none');
-		$tp.$minutesText.prev().css('display', 'none');
-		$tp.$seconds.parent().css('display', 'none');
+	send_test_email(frm) {
+		let d = new frappe.ui.Dialog({
+			title: __('Send Test Email'),
+			fields: [
+				{
+					label: __('Email'),
+					fieldname: 'email',
+					fieldtype: 'Data',
+					options: 'Email',
+				}
+			],
+			primary_action_label: __('Send'),
+			primary_action({ email }) {
+				d.get_primary_btn().text(__('Sending...')).prop('disabled', true);
+				frm.call('send_test_email', { email })
+					.then(() => {
+						d.get_primary_btn().text(__('Send again')).prop('disabled', false);
+					});
+			}
+		});
+		d.show();
 	},
 
 	setup_dashboard(frm) {
-		if(!frm.doc.__islocal && cint(frm.doc.email_sent)
+		if (!frm.doc.__islocal && cint(frm.doc.email_sent)
 			&& frm.doc.__onload && frm.doc.__onload.status_count) {
 			var stat = frm.doc.__onload.status_count;
 			var total = frm.doc.scheduled_to_send;
-			if(total) {
-				$.each(stat, function(k, v) {
+			if (total) {
+				$.each(stat, function (k, v) {
 					stat[k] = flt(v * 100 / total, 2) + '%';
 				});
 
@@ -94,5 +162,58 @@ frappe.ui.form.on('Newsletter', {
 				]);
 			}
 		}
+	},
+
+	setup_sending_status(frm) {
+		frm.call('get_sending_status').then(r => {
+			if (r.message) {
+				frm.events.update_sending_progress(frm, r.message.sent, r.message.total);
+			}
+			if (r.message.sent >= r.message.total) {
+				return;
+			}
+			if (frm.sending_status) return;
+
+			frm.sending_status = setInterval(() => {
+				if (frm.doc.email_sent && frm.$wrapper.is(':visible')) {
+					frm.call('get_sending_status').then(r => {
+						if (r.message) {
+							let { sent, total } = r.message;
+							frm.events.update_sending_progress(frm, sent, total);
+
+							if (sent >= total) {
+								clearInterval(frm.sending_status);
+								frm.sending_status = null;
+								return;
+							}
+						}
+					});
+				}
+			}, 5000);
+		});
+	},
+
+	update_sending_progress(frm, sent, total) {
+		if (sent >= total) {
+			frm.dashboard.hide_progress();
+			return;
+		}
+		frm.dashboard.show_progress(__('Sending emails'), sent * 100 / total, __("{0} of {1} sent", [sent, total]));
+	},
+
+	on_hide(frm) {
+		if (frm.sending_status) {
+			clearInterval(frm.sending_status);
+			frm.sending_status = null;
+		}
+	},
+
+	update_schedule_message(frm) {
+		if (!frm.doc.email_sent && frm.doc.schedule_send) {
+			let datetime = frappe.datetime.global_date_format(frm.doc.schedule_send);
+			frm.dashboard.set_headline_alert(__('This newsletter is scheduled to be sent on {0}', [datetime.bold()]));
+		} else {
+			frm.dashboard.clear_headline();
+		}
 	}
 });

frappe/email/doctype/newsletter/newsletter.json

@@ -7,48 +7,59 @@
  "document_type": "Other",
  "engine": "InnoDB",
  "field_order": [
+  "status_section",
+  "email_sent_at",
+  "column_break_3",
+  "total_recipients",
+  "column_break_12",
+  "email_sent",
+  "from_section",
+  "sender_name",
+  "column_break_5",
+  "sender_email",
+  "column_break_7",
   "send_from",
-  "schedule_sending",
-  "schedule_send",
   "recipients",
   "email_group",
-  "email_sent",
-  "newsletter_content",
+  "subject_section",
   "subject",
+  "newsletter_content",
   "content_type",
   "message",
   "message_md",
   "message_html",
-  "section_break_13",
+  "attachments",
   "send_unsubscribe_link",
-  "send_attachments",
-  "column_break_9",
-  "published",
   "send_webview_link",
-  "route",
-  "test_the_newsletter",
-  "test_email_id",
-  "test_send",
-  "scheduled_to_send"
+  "schedule_settings_section",
+  "scheduled_to_send",
+  "schedule_sending",
+  "schedule_send",
+  "publish_as_a_web_page_section",
+  "published",
+  "route"
  ],
  "fields": [
   {
    "fieldname": "email_group",
    "fieldtype": "Table",
    "in_standard_filter": 1,
-   "label": "Email Group",
-   "options": "Newsletter Email Group"
+   "label": "Audience",
+   "options": "Newsletter Email Group",
+   "reqd": 1
   },
   {
    "fieldname": "send_from",
    "fieldtype": "Data",
    "ignore_xss_filter": 1,
-   "label": "Sender"
+   "label": "Sender",
+   "read_only": 1
   },
   {
    "default": "0",
    "fieldname": "email_sent",
    "fieldtype": "Check",
+   "hidden": 1,
    "label": "Email Sent",
    "no_copy": 1,
    "read_only": 1
@@ -87,32 +98,12 @@
    "label": "Published"
   },
   {
+   "depends_on": "published",
    "fieldname": "route",
    "fieldtype": "Data",
-   "hidden": 1,
    "label": "Route",
    "read_only": 1
   },
-  {
-   "collapsible": 1,
-   "fieldname": "test_the_newsletter",
-   "fieldtype": "Section Break",
-   "label": "Testing"
-  },
-  {
-   "description": "A Lead with this Email Address should exist",
-   "fieldname": "test_email_id",
-   "fieldtype": "Data",
-   "label": "Test Email Address",
-   "options": "Email"
-  },
-  {
-   "depends_on": "eval: doc.test_email_id",
-   "fieldname": "test_send",
-   "fieldtype": "Button",
-   "label": "Test",
-   "options": "test_send"
-  },
   {
    "fieldname": "scheduled_to_send",
    "fieldtype": "Int",
@@ -122,21 +113,16 @@
   {
    "fieldname": "recipients",
    "fieldtype": "Section Break",
-   "label": "Recipients"
+   "label": "To"
   },
   {
    "depends_on": "eval: doc.schedule_sending",
    "fieldname": "schedule_send",
    "fieldtype": "Datetime",
-   "label": "Schedule Send",
+   "label": "Send Email At",
+   "read_only": 1,
    "read_only_depends_on": "eval: doc.email_sent"
   },
-  {
-   "default": "0",
-   "fieldname": "send_attachments",
-   "fieldtype": "Check",
-   "label": "Send Attachments"
-  },
   {
    "fieldname": "content_type",
    "fieldtype": "Select",
@@ -161,23 +147,87 @@
    "default": "0",
    "fieldname": "schedule_sending",
    "fieldtype": "Check",
-   "label": "Schedule Sending",
+   "label": "Schedule sending at a later time",
    "read_only_depends_on": "eval: doc.email_sent"
   },
-  {
-   "fieldname": "column_break_9",
-   "fieldtype": "Column Break"
-  },
   {
    "default": "0",
-   "depends_on": "published",
    "fieldname": "send_webview_link",
    "fieldtype": "Check",
    "label": "Send Web View Link"
   },
   {
-   "fieldname": "section_break_13",
-   "fieldtype": "Section Break"
+   "fieldname": "from_section",
+   "fieldtype": "Section Break",
+   "label": "From"
+  },
+  {
+   "fieldname": "sender_name",
+   "fieldtype": "Data",
+   "label": "Sender Name"
+  },
+  {
+   "fieldname": "sender_email",
+   "fieldtype": "Data",
+   "label": "Sender Email",
+   "options": "Email",
+   "reqd": 1
+  },
+  {
+   "fieldname": "column_break_5",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "column_break_7",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "subject_section",
+   "fieldtype": "Section Break",
+   "label": "Subject"
+  },
+  {
+   "fieldname": "publish_as_a_web_page_section",
+   "fieldtype": "Section Break",
+   "label": "Publish as a web page"
+  },
+  {
+   "depends_on": "schedule_sending",
+   "fieldname": "schedule_settings_section",
+   "fieldtype": "Section Break",
+   "label": "Scheduled Sending"
+  },
+  {
+   "fieldname": "attachments",
+   "fieldtype": "Table",
+   "label": "Attachments",
+   "options": "Newsletter Attachment"
+  },
+  {
+   "fieldname": "email_sent_at",
+   "fieldtype": "Datetime",
+   "label": "Email Sent At",
+   "read_only": 1
+  },
+  {
+   "fieldname": "total_recipients",
+   "fieldtype": "Int",
+   "label": "Total Recipients",
+   "read_only": 1
+  },
+  {
+   "depends_on": "email_sent",
+   "fieldname": "status_section",
+   "fieldtype": "Section Break",
+   "label": "Status"
+  },
+  {
+   "fieldname": "column_break_12",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "column_break_3",
+   "fieldtype": "Column Break"
   }
  ],
  "has_web_view": 1,
@@ -187,7 +237,7 @@
  "is_published_field": "published",
  "links": [],
  "max_attachments": 3,
- "modified": "2021-02-22 14:33:56.095380",
+ "modified": "2021-12-06 20:09:37.963141",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",

frappe/email/doctype/newsletter/newsletter.py

@@ -15,13 +15,11 @@ from .exceptions import NewsletterAlreadySentError, NoRecipientFoundError, Newsl
 
 
 class Newsletter(WebsiteGenerator):
-	def onload(self):
-		self.setup_newsletter_status()
-
 	def validate(self):
 		self.route = f"newsletters/{self.name}"
 		self.validate_sender_address()
 		self.validate_recipient_address()
+		self.validate_publishing()
 
 	@property
 	def newsletter_recipients(self) -> List[str]:
@@ -30,29 +28,55 @@ class Newsletter(WebsiteGenerator):
 		return self._recipients
 
 	@frappe.whitelist()
-	def test_send(self):
-		test_emails = frappe.utils.split_emails(self.test_email_id)
-		self.queue_all(test_emails=test_emails)
-		frappe.msgprint(_("Test email sent to {0}").format(self.test_email_id))
+	def get_sending_status(self):
+		count_by_status = frappe.get_all("Email Queue",
+			filters={"reference_doctype": self.doctype, "reference_name": self.name},
+			fields=["status", "count(name) as count"],
+			group_by="status",
+			order_by="status"
+		)
+		sent = 0
+		total = 0
+		for row in count_by_status:
+			if row.status == "Sent":
+				sent = row.count
+			total += row.count
+
+		return {'sent': sent, 'total': total}
+
+	@frappe.whitelist()
+	def send_test_email(self, email):
+		test_emails = frappe.utils.validate_email_address(email, throw=True)
+		self.send_newsletter(emails=test_emails)
+		frappe.msgprint(_("Test email sent to {0}").format(email), alert=True)
+
+	@frappe.whitelist()
+	def find_broken_links(self):
+		from bs4 import BeautifulSoup
+		import requests
+
+		html = self.get_message()
+		soup = BeautifulSoup(html, "html.parser")
+		links = soup.find_all("a")
+		images = soup.find_all("img")
+		broken_links = []
+		for el in links + images:
+			url = el.attrs.get("href") or el.attrs.get("src")
+			try:
+				response = requests.head(url, verify=False, timeout=5)
+				if response.status_code >= 400:
+					broken_links.append(url)
+			except:
+				broken_links.append(url)
+		return broken_links
 
 	@frappe.whitelist()
 	def send_emails(self):
-		"""send emails to leads and customers"""
+		"""queue sending emails to recipients"""
+		self.schedule_sending = False
+		self.schedule_send = None
 		self.queue_all()
-		frappe.msgprint(_("Email queued to {0} recipients").format(len(self.newsletter_recipients)))
-
-	def setup_newsletter_status(self):
-		"""Setup analytical status for current Newsletter. Can be accessible from desk.
-		"""
-		if self.email_sent:
-			status_count = frappe.get_all("Email Queue",
-				filters={"reference_doctype": self.doctype, "reference_name": self.name},
-				fields=["status", "count(name)"],
-				group_by="status",
-				order_by="status",
-				as_list=True,
-			)
-			self.get("__onload").status_count = dict(status_count)
+		frappe.msgprint(_("Email queued to {0} recipients").format(self.total_recipients))
 
 	def validate_send(self):
 		"""Validate if Newsletter can be sent.
@@ -75,8 +99,9 @@ class Newsletter(WebsiteGenerator):
 	def validate_sender_address(self):
 		"""Validate self.send_from is a valid email address or not.
 		"""
-		if self.send_from:
-			frappe.utils.validate_email_address(self.send_from, throw=True)
+		if self.sender_email:
+			frappe.utils.validate_email_address(self.sender_email, throw=True)
+			self.send_from = f"{self.sender_name} <{self.sender_email}>" if self.sender_name else self.sender_email
 
 	def validate_recipient_address(self):
 		"""Validate if self.newsletter_recipients are all valid email addresses or not.
@@ -84,6 +109,10 @@ class Newsletter(WebsiteGenerator):
 		for recipient in self.newsletter_recipients:
 			frappe.utils.validate_email_address(recipient, throw=True)
 
+	def validate_publishing(self):
+		if self.send_webview_link and not self.published:
+			frappe.throw(_("Newsletter must be published to send webview link in email"))
+
 	def get_linked_email_queue(self) -> List[str]:
 		"""Get list of email queue linked to this newsletter.
 		"""
@@ -116,45 +145,24 @@ class Newsletter(WebsiteGenerator):
 			x for x in self.newsletter_recipients if x not in self.get_success_recipients()
 		]
 
-	def queue_all(self, test_emails: List[str] = None):
-		"""Queue Newsletter to all the recipients generated from the `Email Group`
-		table
-
-		Args:
-			test_email (List[str], optional): Send test Newsletter to the passed set of emails.
-			Defaults to None.
+	def queue_all(self):
+		"""Queue Newsletter to all the recipients generated from the `Email Group` table
 		"""
-		if test_emails:
-			for test_email in test_emails:
-				frappe.utils.validate_email_address(test_email, throw=True)
-		else:
-			self.validate()
-			self.validate_send()
+		self.validate()
+		self.validate_send()
 
-		newsletter_recipients = test_emails or self.get_pending_recipients()
-		self.send_newsletter(emails=newsletter_recipients)
+		recipients = self.get_pending_recipients()
+		self.send_newsletter(emails=recipients)
 
-		if not test_emails:
-			self.email_sent = True
-			self.schedule_send = frappe.utils.now_datetime()
-			self.scheduled_to_send = len(newsletter_recipients)
-			self.save()
+		self.email_sent = True
+		self.email_sent_at = frappe.utils.now()
+		self.total_recipients = len(recipients)
+		self.save()
 
 	def get_newsletter_attachments(self) -> List[Dict[str, str]]:
 		"""Get list of attachments on current Newsletter
 		"""
-		attachments = []
-
-		if self.send_attachments:
-			files = frappe.get_all(
-				"File",
-				filters={"attached_to_doctype": "Newsletter", "attached_to_name": self.name},
-				order_by="creation desc",
-				pluck="name",
-			)
-			attachments.extend({"fid": file} for file in files)
-
-		return attachments
+		return [{"file_url": row.attachment} for row in self.attachments]
 
 	def send_newsletter(self, emails: List[str]):
 		"""Trigger email generation for `emails` and add it in Email Queue.
@@ -186,12 +194,13 @@ class Newsletter(WebsiteGenerator):
 		frappe.db.auto_commit_on_many_writes = is_auto_commit_set
 
 	def get_message(self) -> str:
-		if self.content_type == "HTML":
-			return frappe.render_template(self.message_html, {"doc": self.as_dict()})
+		message = self.message
 		if self.content_type == "Markdown":
-			return frappe.utils.markdown(self.message_md)
-		# fallback to Rich Text
-		return self.message
+			message = frappe.utils.md_to_html(self.message_md)
+		if self.content_type == "HTML":
+			message = self.message_html
+
+		return frappe.render_template(message, {"doc": self.as_dict()})
 
 	def get_recipients(self) -> List[str]:
 		"""Get recipients from Email Group"""
@@ -223,21 +232,6 @@ class Newsletter(WebsiteGenerator):
 			},
 		)
 
-	def get_context(self, context):
-		newsletters = get_newsletter_list("Newsletter", None, None, 0)
-		if newsletters:
-			newsletter_list = [d.name for d in newsletters]
-			if self.name not in newsletter_list:
-				frappe.redirect_to_message(
-					_("Permission Error"), _("You are not permitted to view the newsletter.")
-				)
-				frappe.local.flags.redirect_location = frappe.local.response.location
-				raise frappe.Redirect
-			else:
-				context.attachments = self.get_attachments()
-		context.no_cache = 1
-		context.show_sidebar = True
-
 
 @frappe.whitelist(allow_guest=True)
 def confirmed_unsubscribe(email, group):
@@ -320,35 +314,14 @@ def confirm_subscription(email, email_group=_("Website")):
 
 def get_list_context(context=None):
 	context.update({
-		"show_sidebar": True,
 		"show_search": True,
-		'no_breadcrumbs': True,
-		"title": _("Newsletter"),
-		"get_list": get_newsletter_list,
+		"no_breadcrumbs": True,
+		"title": _("Newsletters"),
+		"filters": {"published": 1},
 		"row_template": "email/doctype/newsletter/templates/newsletter_row.html",
 	})
 
 
-def get_newsletter_list(doctype, txt, filters, limit_start, limit_page_length=20, order_by="modified"):
-	email_group_list = frappe.db.sql('''SELECT eg.name
-		FROM `tabEmail Group` eg, `tabEmail Group Member` egm
-		WHERE egm.unsubscribed=0
-		AND eg.name=egm.email_group
-		AND egm.email = %s''', frappe.session.user)
-	email_group_list = [d[0] for d in email_group_list]
-
-	if email_group_list:
-		return frappe.db.sql('''SELECT n.name, n.subject, n.message, n.modified
-			FROM `tabNewsletter` n, `tabNewsletter Email Group` neg
-			WHERE n.name = neg.parent
-			AND n.email_sent=1
-			AND n.published=1
-			AND neg.email_group in ({0})
-			ORDER BY n.modified DESC LIMIT {1} OFFSET {2}
-			'''.format(','.join(['%s'] * len(email_group_list)),
-					limit_page_length, limit_start), email_group_list, as_dict=1)
-
-
 def send_scheduled_email():
 	"""Send scheduled newsletter to the recipients."""
 	scheduled_newsletter = frappe.get_all(

frappe/email/doctype/newsletter/templates/newsletter.html

@@ -1,6 +1,6 @@
 {% extends "templates/web.html" %}
 
-{% block title %} {{ _("Newsletter") }} {% endblock %}
+{% block title %} {{ doc.subject }} {% endblock %}
 
 {% block page_content %}
     <style>
@@ -36,11 +36,11 @@
                 </p>
             </div>
             <div itemprop="articleBody" class="longform blog-text">
-                {{ doc.message }}
+                {{ doc.get_message() }}
             </div>
         </article>
 
-        {% if attachments %}
+        {% if doc.attachments %}
             <div>
                 <div class="row text-muted">
                     <div class="col-sm-12 h6 text-uppercase">
@@ -49,10 +49,10 @@
                 </div>
                 <div class="row">
                     <div class="col-sm-12">
-                        {% for attachment in attachments %}
+                        {% for attachment in doc.attachments %}
                             <p class="small">
-                                <a href="{{ attachment.file_url }}" target="blank"> 
-                                    {{ attachment.file_name }}
+                                <a href="{{ attachment.attachment }}" target="_blank">
+                                    {{ attachment.attachment }}
                                 </a>
                             </p>
                         {% endfor %}

frappe/email/doctype/newsletter/test_newsletter.py

@@ -14,7 +14,6 @@ from frappe.email.doctype.newsletter.exceptions import (
 from frappe.email.doctype.newsletter.newsletter import (
 	Newsletter,
 	confirmed_unsubscribe,
-	get_newsletter_list,
 	send_scheduled_email
 )
 from frappe.email.queue import flush
@@ -101,7 +100,8 @@ class TestNewsletterMixin:
 		doctype = "Newsletter"
 		newsletter_content = {
 			"subject": "_Test Newsletter",
-			"send_from": "Test Sender <test_sender@example.com>",
+			"sender_name": "Test Sender",
+			"sender_email": "test_sender@example.com",
 			"content_type": "Rich Text",
 			"message": "Testing my news.",
 		}
@@ -157,21 +157,6 @@ class TestNewsletter(TestNewsletterMixin, unittest.TestCase):
 			if email != to_unsubscribe:
 				self.assertTrue(email in recipients)
 
-	def test_portal(self):
-		self.send_newsletter(published=1)
-		frappe.set_user("test1@example.com")
-		newsletter_list = get_newsletter_list("Newsletter", None, None, 0)
-		self.assertEqual(len(newsletter_list), 1)
-
-	def test_newsletter_context(self):
-		context = frappe._dict()
-		newsletter_name = self.send_newsletter(published=1)
-		frappe.set_user("test2@example.com")
-		doc = frappe.get_doc("Newsletter", newsletter_name)
-		doc.get_context(context)
-		self.assertEqual(context.no_cache, 1)
-		self.assertTrue("attachments" not in list(context))
-
 	def test_schedule_send(self):
 		self.send_newsletter(schedule_send=add_days(getdate(), -1))
 
@@ -181,26 +166,32 @@ class TestNewsletter(TestNewsletterMixin, unittest.TestCase):
 		for email in emails:
 			self.assertTrue(email in recipients)
 
-	def test_newsletter_test_send(self):
-		"""Test "Test Send" functionality of Newsletter
+	def test_newsletter_send_test_email(self):
+		"""Test "Send Test Email" functionality of Newsletter
 		"""
 		newsletter = self.get_newsletter()
-		newsletter.test_email_id = choice(emails)
-		newsletter.test_send()
+		test_email = choice(emails)
+		newsletter.send_test_email(test_email)
 
 		self.assertFalse(newsletter.email_sent)
 		newsletter.save = MagicMock()
 		self.assertFalse(newsletter.save.called)
+		# check if the test email is in the queue
+		email_queue = frappe.db.get_all('Email Queue', filters=[
+			['reference_doctype', '=', 'Newsletter'],
+			['reference_name', '=', newsletter.name],
+			['Email Queue Recipient', 'recipient', '=', test_email]
+		])
+		self.assertTrue(email_queue)
 
 	def test_newsletter_status(self):
 		"""Test for Newsletter's stats on onload event
 		"""
 		newsletter = self.get_newsletter()
 		newsletter.email_sent = True
-		# had to use run_onload as calling .onload directly bought weird errors
-		# like TestNewsletter has no attribute "_TestNewsletter__onload"
-		run_onload(newsletter)
-		self.assertIsInstance(newsletter.get("__onload").status_count, dict)
+		result = newsletter.get_sending_status()
+		self.assertTrue('total' in result)
+		self.assertTrue('sent' in result)
 
 	def test_already_sent_newsletter(self):
 		newsletter = self.get_newsletter()
@@ -218,22 +209,6 @@ class TestNewsletter(TestNewsletterMixin, unittest.TestCase):
 			with self.assertRaises(NoRecipientFoundError):
 				newsletter.send_emails()
 
-	def test_send_newsletter_with_attachments(self):
-		newsletter = self.get_newsletter()
-		newsletter.reload()
-		file_attachment = frappe.get_doc({
-			"doctype": "File",
-			"file_name": "test1.txt",
-			"attached_to_doctype": newsletter.doctype,
-			"attached_to_name": newsletter.name,
-			"content": frappe.mock("paragraph")
-		})
-		file_attachment.save()
-		newsletter.send_attachments = True
-		newsletter_attachments = newsletter.get_newsletter_attachments()
-		self.assertEqual(len(newsletter_attachments), 1)
-		self.assertEqual(newsletter_attachments[0]["fid"], file_attachment.name)
-
 	def test_send_scheduled_email_error_handling(self):
 		newsletter = self.get_newsletter(schedule_send=add_days(getdate(), -1))
 		job_path = "frappe.email.doctype.newsletter.newsletter.Newsletter.queue_all"

frappe/email/doctype/newsletter_attachment/newsletter_attachment.json

@@ -0,0 +1,31 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2021-12-06 16:37:40.652468",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "attachment"
+ ],
+ "fields": [
+  {
+   "fieldname": "attachment",
+   "fieldtype": "Attach",
+   "in_list_view": 1,
+   "label": "Attachment",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-12-06 16:37:47.481057",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Newsletter Attachment",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC"
+}

frappe/email/doctype/newsletter_attachment/newsletter_attachment.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class NewsletterAttachment(Document):
+	pass

frappe/email/doctype/newsletter_email_group/newsletter_email_group.json

@@ -1,106 +1,42 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0,
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-02-26 16:20:52.654136", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2017-02-26 16:20:52.654136",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "email_group",
+  "total_subscribers"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "email_group", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Email Group", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Email Group", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0,
-   "unique": 0
-  }, 
+   "columns": 7,
+   "fieldname": "email_group",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Email Group",
+   "options": "Email Group",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
+   "columns": 3,
    "fetch_from": "email_group.total_subscribers",
-   "fieldname": "total_subscribers", 
-   "fieldtype": "Read Only", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Total Subscribers", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0,
-   "unique": 0
+   "fieldname": "total_subscribers",
+   "fieldtype": "Read Only",
+   "in_list_view": 1,
+   "label": "Total Subscribers"
   }
- ], 
- "has_web_view": 0,
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2018-05-16 22:42:55.437367",
- "modified_by": "Administrator", 
- "module": "Email", 
- "name": "Newsletter Email Group", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2021-12-06 20:12:08.420240",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Newsletter Email Group",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
 }

frappe/email/receive.py

@@ -111,7 +111,17 @@ class EmailServer:
 				frappe.msgprint(_('Invalid User Name or Support Password. Please rectify and try again.'))
 				raise
 
-	def get_messages(self):
+	def select_imap_folder(self, folder):
+		self.imap.select(folder)
+
+	def logout(self):
+		if cint(self.settings.use_imap):
+			self.imap.logout()
+		else:
+			self.pop.quit()
+		return
+
+	def get_messages(self, folder="INBOX"):
 		"""Returns new email messages in a list."""
 		if not (self.check_mails() or self.connect()):
 			return []
@@ -126,7 +136,8 @@ class EmailServer:
 			self.latest_messages = []
 			self.seen_status = {}
 			self.uid_reindexed = False
-			uid_list = email_list = self.get_new_mails()
+
+			uid_list = email_list = self.get_new_mails(folder)
 
 			if not email_list:
 				return
@@ -160,13 +171,6 @@ class EmailServer:
 			else:
 				raise
 
-		finally:
-			# no matter the exception, pop should quit if connected
-			if cint(self.settings.use_imap):
-				self.imap.logout()
-			else:
-				self.pop.quit()
-
 		out = { "latest_messages": self.latest_messages }
 		if self.settings.use_imap:
 			out.update({
@@ -177,15 +181,15 @@ class EmailServer:
 
 		return out
 
-	def get_new_mails(self):
+	def get_new_mails(self, folder):
 		"""Return list of new mails"""
 		if cint(self.settings.use_imap):
 			email_list = []
-			self.check_imap_uidvalidity()
+			self.check_imap_uidvalidity(folder)
 
 			readonly = False if self.settings.email_sync_rule == "UNSEEN" else True
 
-			self.imap.select("Inbox", readonly=readonly)
+			self.imap.select(folder, readonly=readonly)
 			response, message = self.imap.uid('search', None, self.settings.email_sync_rule)
 			if message[0]:
 				email_list =  message[0].split()
@@ -194,11 +198,11 @@ class EmailServer:
 
 		return email_list
 
-	def check_imap_uidvalidity(self):
+	def check_imap_uidvalidity(self, folder):
 		# compare the UIDVALIDITY of email account and imap server
 		uid_validity = self.settings.uid_validity
 
-		response, message = self.imap.status("Inbox", "(UIDVALIDITY UIDNEXT)")
+		response, message = self.imap.status(folder, "(UIDVALIDITY UIDNEXT)")
 		current_uid_validity = self.parse_imap_response("UIDVALIDITY", message[0]) or 0
 
 		uidnext = int(self.parse_imap_response("UIDNEXT", message[0]) or "1")
@@ -206,14 +210,26 @@ class EmailServer:
 
 		if not uid_validity or uid_validity != current_uid_validity:
 			# uidvalidity changed & all email uids are reindexed by server
-			frappe.db.sql(
-				"""update `tabCommunication` set uid=-1 where communication_medium='Email'
-				and email_account=%s""", (self.settings.email_account,)
-			)
-			frappe.db.sql(
-				"""update `tabEmail Account` set uidvalidity=%s, uidnext=%s where
-				name=%s""", (current_uid_validity, uidnext, self.settings.email_account)
-			)
+			Communication = frappe.qb.DocType("Communication")
+			frappe.qb.update(Communication) \
+				.set(Communication.uid, -1) \
+				.where(Communication.communication_medium == "Email") \
+				.where(Communication.email_account == self.settings.email_account).run()
+
+			if self.settings.use_imap:
+				# new update for the IMAP Folder DocType
+				IMAPFolder = frappe.qb.DocType("IMAP Folder")
+				frappe.qb.update(IMAPFolder) \
+					.set(IMAPFolder.uidvalidity, current_uid_validity) \
+					.set(IMAPFolder.uidnext, uidnext) \
+					.where(IMAPFolder.parent == self.settings.email_account_name) \
+					.where(IMAPFolder.folder_name == folder).run()
+			else:
+				EmailAccount = frappe.qb.DocType("Email Account")
+				frappe.qb.update(EmailAccount) \
+					.set(EmailAccount.uidvalidity, current_uid_validity) \
+					.set(EmailAccount.uidnext, uidnext) \
+					.where(EmailAccount.name == self.settings.email_account_name).run()
 
 			# uid validity not found pulling emails for first time
 			if not uid_validity:
@@ -232,6 +248,7 @@ class EmailServer:
 	def parse_imap_response(self, cmd, response):
 		pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd)
 		match = re.search(pattern, response.decode('utf-8'), re.U | re.I)
+
 		if match:
 			return match.group(0)
 		else:
@@ -340,16 +357,15 @@ class EmailServer:
 
 		return error_msg
 
-	def update_flag(self, uid_list=None):
+	def update_flag(self, folder, uid_list=None):
 		""" set all uids mails the flag as seen  """
-
 		if not uid_list:
 			return
 
 		if not self.connect():
 			return
 
-		self.imap.select("Inbox")
+		self.imap.select(folder)
 		for uid, operation in uid_list.items():
 			if not uid: continue
 

frappe/email/test_smtp.py

@@ -16,11 +16,12 @@ class TestSMTP(unittest.TestCase):
 			make_server(port, 0, 1)
 
 	def test_get_email_account(self):
-		existing_email_accounts = frappe.get_all("Email Account", fields = ["name", "enable_outgoing", "default_outgoing", "append_to"])
+		existing_email_accounts = frappe.get_all("Email Account", fields = ["name", "enable_outgoing", "default_outgoing","append_to", "use_imap"])
 		unset_details = {
 			"enable_outgoing": 0,
 			"default_outgoing": 0,
-			"append_to": None
+			"append_to": None,
+			"use_imap": 0
 		}
 		for email_account in existing_email_accounts:
 			frappe.db.set_value('Email Account', email_account['name'], unset_details)
@@ -60,7 +61,8 @@ def create_email_account(email_id, password, enable_outgoing, default_outgoing=0
 		"enable_incoming": 1,
 		"append_to":append_to,
 		"is_dummy_password": 1,
-		"smtp_server": "localhost"
+		"smtp_server": "localhost",
+		"use_imap": 0
 	}
 
 	email_account = frappe.new_doc('Email Account')

frappe/event_streaming/doctype/event_producer/event_producer.py

@@ -54,6 +54,11 @@ class EventProducer(Document):
 			self.db_set('incoming_change', 0)
 			self.reload()
 
+	def on_trash(self):
+		last_update = frappe.db.get_value('Event Producer Last Update', dict(event_producer=self.name))
+		if last_update:
+			frappe.delete_doc('Event Producer Last Update', last_update)
+
 	def check_url(self):
 		valid_url_schemes = ("http", "https")
 		frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes)

frappe/handler.py

@@ -255,7 +255,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
 		response = doc.run_method(method, **args)
 
 	frappe.response.docs.append(doc)
-	if not response:
+	if response is None:
 		return
 
 	# build output as csv

frappe/hooks.py

@@ -240,7 +240,8 @@ scheduler_events = {
 		"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
 		"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
 		"frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports",
-		"frappe.core.doctype.log_settings.log_settings.run_log_clean_up"
+		"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",
+		"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.process_data_deletion_request"
 	],
 	"daily_long": [
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",

frappe/installer.py

@@ -324,7 +324,7 @@ def _delete_doctypes(doctypes: List[str], dry_run: bool) -> None:
 		print(f"* dropping Table for '{doctype}'...")
 		if not dry_run:
 			frappe.delete_doc("DocType", doctype, ignore_on_trash=True)
-			frappe.db.sql_ddl(f"drop table `tab{doctype}`")
+			frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{doctype}`")
 
 
 def post_install(rebuild_website=False):

frappe/model/__init__.py

@@ -38,6 +38,11 @@ data_fieldtypes = (
 	'Icon'
 )
 
+attachment_fieldtypes = (
+	'Attach',
+	'Attach Image',
+)
+
 no_value_fields = (
 	'Section Break',
 	'Column Break',

frappe/model/base_document.py

@@ -338,7 +338,7 @@ class BaseDocument(object):
 			return self.meta.get_field(fieldname).options
 		except AttributeError:
 			if self.doctype == 'DocType':
-				return dict(links='DocType Link', actions='DocType Action').get(fieldname)
+				return dict(links='DocType Link', actions='DocType Action', states='DocType State').get(fieldname)
 			raise
 
 	def get_parentfield_of_doctype(self, doctype):