Merge branch 'develop' of github.com:frappe/frappe into try_except_for_comm

cypress/integration/workspace.js

@@ -23,7 +23,7 @@ context('Workspace 2.0', () => {
 		// check if sidebar item is added in pubic section
 		cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0');
 
-		cy.get('.standard-actions .btn-primary[data-label="Save Customizations"]').click();
+		cy.get('.standard-actions .btn-primary[data-label="Save"]').click();
 		cy.wait(300);
 		cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0');
 
@@ -67,7 +67,7 @@ context('Workspace 2.0', () => {
 		cy.get('.ce-block:last .dropdown-item').contains('Expand').click();
 		cy.get(".ce-block:last").should('have.class', 'col-xs-12');
 
-		cy.get('.standard-actions .btn-primary[data-label="Save Customizations"]').click();
+		cy.get('.standard-actions .btn-primary[data-label="Save"]').click();
 	});
 
 	it('Delete Private Page', () => {
@@ -80,7 +80,7 @@ context('Workspace 2.0', () => {
 			.find('.dropdown-item[title="Delete Workspace"]').click({force: true});
 		cy.wait(300);
 		cy.get('.modal-footer > .standard-actions > .btn-modal-primary:visible').first().click();
-		cy.get('.standard-actions .btn-primary[data-label="Save Customizations"]').click();
+		cy.get('.standard-actions .btn-primary[data-label="Save"]').click();
 		cy.get('.codex-editor__redactor .ce-block');
 		cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('not.exist');
 	});

frappe/__init__.py

@@ -358,7 +358,7 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False,
 	response JSON and shown in a pop-up / modal.
 
 	:param msg: Message.
-	:param title: [optional] Message title.
+	:param title: [optional] Message title. Default: "Message".
 	:param raise_exception: [optional] Raise given exception and show message.
 	:param as_table: [optional] If `msg` is a list of lists, render as HTML table.
 	:param as_list: [optional] If `msg` is a list, render as un-ordered list.
@@ -395,8 +395,7 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False,
 	if flags.print_messages and out.message:
 		print(f"Message: {strip_html_tags(out.message)}")
 
-	if title:
-		out.title = title
+	out.title = title or _("Message", context="Default title of the message dialog")
 
 	if not indicator and raise_exception:
 		indicator = 'red'

frappe/api.py

@@ -94,7 +94,8 @@ def handle():
 						"data": doc.save().as_dict()
 					})
 
-					if doc.parenttype and doc.parent:
+					# check for child table doctype
+					if doc.get("parenttype"):
 						frappe.get_doc(doc.parenttype, doc.parent).save()
 
 					frappe.db.commit()

frappe/app.py

@@ -294,7 +294,6 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 	_sites_path = sites_path
 
 	from werkzeug.serving import run_simple
-	patch_werkzeug_reloader()
 
 	if profile or os.environ.get('USE_PROFILER'):
 		application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls'))
@@ -325,23 +324,3 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 		use_debugger=not in_test_env,
 		use_evalex=not in_test_env,
 		threaded=not no_threading)
-
-def patch_werkzeug_reloader():
-	"""
-	This function monkey patches Werkzeug reloader to ignore reloading files in
-	the __pycache__ directory.
-
-	To be deprecated when upgrading to Werkzeug 2.
-	"""
-
-	from werkzeug._reloader import WatchdogReloaderLoop
-
-	trigger_reload = WatchdogReloaderLoop.trigger_reload
-
-	def custom_trigger_reload(self, filename):
-		if os.path.basename(os.path.dirname(filename)) == "__pycache__":
-			return
-
-		return trigger_reload(self, filename)
-
-	WatchdogReloaderLoop.trigger_reload = custom_trigger_reload

frappe/client.py

@@ -128,7 +128,7 @@ def set_value(doctype, name, fieldname, value=None):
 	:param fieldname: fieldname string or JSON / dict with key value pair
 	:param value: value if fieldname is JSON / dict'''
 
-	if fieldname!="idx" and fieldname in frappe.model.default_fields:
+	if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
 		frappe.throw(_("Cannot edit standard fields"))
 
 	if not value:
@@ -141,14 +141,15 @@ def set_value(doctype, name, fieldname, value=None):
 	else:
 		values = {fieldname: value}
 
-	doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
-	if doc and doc.parent and doc.parenttype:
+	# check for child table doctype
+	if not frappe.get_meta(doctype).istable:
+		doc = frappe.get_doc(doctype, name)
+		doc.update(values)
+	else:
+		doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
 		doc = frappe.get_doc(doc.parenttype, doc.parent)
 		child = doc.getone({"doctype": doctype, "name": name})
 		child.update(values)
-	else:
-		doc = frappe.get_doc(doctype, name)
-		doc.update(values)
 
 	doc.save()
 
@@ -162,10 +163,10 @@ def insert(doc=None):
 	if isinstance(doc, str):
 		doc = json.loads(doc)
 
-	if doc.get("parent") and doc.get("parenttype"):
+	if doc.get("parenttype"):
 		# inserting a child record
-		parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
-		parent.append(doc.get("parentfield"), doc)
+		parent = frappe.get_doc(doc.parenttype, doc.parent)
+		parent.append(doc.parentfield, doc)
 		parent.save()
 		return parent.as_dict()
 	else:
@@ -186,10 +187,10 @@ def insert_many(docs=None):
 		frappe.throw(_('Only 200 inserts allowed in one request'))
 
 	for doc in docs:
-		if doc.get("parent") and doc.get("parenttype"):
+		if doc.get("parenttype"):
 			# inserting a child record
-			parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
-			parent.append(doc.get("parentfield"), doc)
+			parent = frappe.get_doc(doc.parenttype, doc.parent)
+			parent.append(doc.parentfield, doc)
 			parent.save()
 			out.append(parent.name)
 		else:

frappe/core/doctype/comment/test_comment.py

@@ -70,6 +70,19 @@ class TestComment(unittest.TestCase):
 			reference_name = test_blog.name
 		))), 0)
 
+		# test for filtering html and css injection elements
+		frappe.db.delete("Comment", {"reference_doctype": "Blog Post"})
+
+		frappe.form_dict.comment = '<script>alert(1)</script>Comment'
+		frappe.form_dict.comment_by = 'hacker'
+
+		add_comment()
+
+		self.assertEqual(frappe.get_all('Comment', fields = ['content'], filters = dict(
+			reference_doctype = test_blog.doctype,
+			reference_name = test_blog.name
+		))[0]['content'], 'Comment')
+
 		test_blog.delete()
 
 

frappe/core/doctype/data_import/importer.py

@@ -618,7 +618,7 @@ class Row:
 			)
 
 		# remove standard fields and __islocal
-		for key in frappe.model.default_fields + ("__islocal",):
+		for key in frappe.model.default_fields + frappe.model.child_table_fields + ("__islocal",):
 			doc.pop(key, None)
 
 		for col, value in zip(columns, values):

frappe/core/doctype/data_import/test_importer.py

@@ -4,6 +4,7 @@
 import unittest
 import frappe
 from frappe.core.doctype.data_import.importer import Importer
+from frappe.tests.test_query_builder import db_type_is, run_only_if
 from frappe.utils import getdate, format_duration
 
 doctype_name = 'DocType for Import'
@@ -54,6 +55,8 @@ class TestImporter(unittest.TestCase):
 		self.assertEqual(len(preview.data), 4)
 		self.assertEqual(len(preview.columns), 16)
 
+	# ignored on postgres because myisam doesn't exist on pg
+	@run_only_if(db_type_is.MARIADB)
 	def test_data_import_without_mandatory_values(self):
 		import_file = get_import_file('sample_import_file_without_mandatory')
 		data_import = self.get_importer(doctype_name, import_file)

frappe/core/doctype/doctype/doctype.py

@@ -10,7 +10,9 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache
 import frappe
 from frappe import _
 from frappe.utils import now, cint
-from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options
+from frappe.model import (
+	no_value_fields, default_fields, table_fields, data_field_options, child_table_fields
+)
 from frappe.model.document import Document
 from frappe.model.base_document import get_controller
 from frappe.custom.doctype.property_setter.property_setter import make_property_setter
@@ -74,6 +76,7 @@ class DocType(Document):
 		self.make_amendable()
 		self.make_repeatable()
 		self.validate_nestedset()
+		self.validate_child_table()
 		self.validate_website()
 		self.ensure_minimum_max_attachment_limit()
 		validate_links_table_fieldnames(self)
@@ -689,6 +692,22 @@ class DocType(Document):
 		})
 		self.nsm_parent_field = parent_field_name
 
+	def validate_child_table(self):
+		if not self.get("istable") or self.is_new():
+			# if the doctype is not a child table then return
+			# if the doctype is a new doctype and also a child table then
+			# don't move forward as it will be handled via schema
+			return
+
+		self.add_child_table_fields()
+
+	def add_child_table_fields(self):
+		from frappe.database.schema import add_column
+
+		add_column(self.name, "parent", "Data")
+		add_column(self.name, "parenttype", "Data")
+		add_column(self.name, "parentfield", "Data")
+
 	def get_max_idx(self):
 		"""Returns the highest `idx`"""
 		max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""",
@@ -1016,7 +1035,7 @@ def validate_fields(meta):
 				sort_fields = [d.split()[0] for d in meta.sort_field.split(',')]
 
 			for fieldname in sort_fields:
-				if not fieldname in fieldname_list + list(default_fields):
+				if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)):
 					frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname),
 						InvalidFieldNameError)
 

frappe/core/doctype/file/file.py

@@ -878,8 +878,9 @@ def extract_images_from_html(doc, content, is_private=False):
 		else:
 			filename = get_random_filename(content_type=mtype)
 
-		doctype = doc.parenttype if doc.parent else doc.doctype
-		name = doc.parent or doc.name
+		# attaching a file to a child table doc, attaches it to the parent doc
+		doctype = doc.parenttype if doc.get("parent") else doc.doctype
+		name = doc.get("parent") or doc.name
 
 		_file = frappe.get_doc({
 			"doctype": "File",

frappe/core/doctype/file/test_file.py

@@ -406,7 +406,7 @@ class TestFile(unittest.TestCase):
 		test_file.reload()
 		test_file.file_url = frappe.utils.get_url('unknown.jpg')
 		test_file.make_thumbnail(suffix="xs")
-		self.assertEqual(json.loads(frappe.message_log[0]), {"message": f"File '{frappe.utils.get_url('unknown.jpg')}' not found"})
+		self.assertEqual(json.loads(frappe.message_log[0]).get("message"), f"File '{frappe.utils.get_url('unknown.jpg')}' not found")
 		self.assertEquals(test_file.thumbnail_url, None)
 
 	def test_file_unzip(self):

frappe/core/doctype/report/report.py

@@ -61,7 +61,7 @@ class Report(Document):
 					delete_permanently=True)
 
 	def get_columns(self):
-		return [d.as_dict(no_default_fields = True) for d in self.columns]
+		return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
 
 	@frappe.whitelist()
 	def set_doctype_roles(self):

frappe/core/doctype/server_script/test_server_script.py

@@ -139,3 +139,42 @@ class TestServerScript(unittest.TestCase):
 
 		server_script.disabled = 1
 		server_script.save()
+
+	def test_restricted_qb(self):
+		todo = frappe.get_doc(doctype="ToDo", description="QbScriptTestNote")
+		todo.insert()
+
+		script = frappe.get_doc(
+			doctype='Server Script',
+			name='test_qb_restrictions',
+			script_type = 'API',
+			api_method = 'test_qb_restrictions',
+			allow_guest = 1,
+			# whitelisted update
+			script = f'''
+frappe.db.set_value("ToDo", "{todo.name}", "description", "safe")
+'''
+		)
+		script.insert()
+		script.execute_method()
+
+		todo.reload()
+		self.assertEqual(todo.description, "safe")
+
+		# unsafe update
+		script.script = f"""
+todo = frappe.qb.DocType("ToDo")
+frappe.qb.update(todo).set(todo.description, "unsafe").where(todo.name == "{todo.name}").run()
+"""
+		script.save()
+		self.assertRaises(frappe.PermissionError, script.execute_method)
+		todo.reload()
+		self.assertEqual(todo.description, "safe")
+
+		# safe select
+		script.script = f"""
+todo = frappe.qb.DocType("ToDo")
+frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
+"""
+		script.save()
+		script.execute_method()

frappe/core/doctype/user/test_user.py

@@ -359,6 +359,7 @@ class TestUser(unittest.TestCase):
 				json.loads(frappe.message_log[0]).get("message"), 
 				"Password reset instructions have been sent to your email"
 			)
+
 		sendmail.assert_called_once()
 		self.assertEqual(sendmail.call_args[1]["recipients"], "test2@example.com")
 

frappe/core/doctype/user_type/user_type.py

@@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters
 		['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]]
 
 	doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters,
-		order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1)
+		order_by='`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1)
 
 	custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)],
 		['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']]

frappe/core/notifications.py

@@ -39,43 +39,3 @@ def get_todays_events(as_list=False):
 	today = nowdate()
 	events = get_events(today, today)
 	return events if as_list else len(events)
-
-def get_unseen_likes():
-	"""Returns count of unseen likes"""
-
-	comment_doctype = DocType("Comment")
-	return frappe.db.count(comment_doctype,
-		filters=(
-			(comment_doctype.comment_type == "Like")
-			& (comment_doctype.modified >= Now() - Interval(years=1))
-			& (comment_doctype.owner.notnull())
-			& (comment_doctype.owner != frappe.session.user)
-			& (comment_doctype.reference_owner == frappe.session.user)
-			& (comment_doctype.seen == 0)
-		)
-	)
-
-
-def get_unread_emails():
-	"returns count of unread emails for a user"
-
-	communication_doctype = DocType("Communication")
-	user_doctype = DocType("User")
-	distinct_email_accounts = (
-		frappe.qb.from_(user_doctype)
-		.select(user_doctype.email_account)
-		.where(user_doctype.parent == frappe.session.user)
-		.distinct()
-	)
-
-	return frappe.db.count(communication_doctype,
-		filters=(
-			(communication_doctype.communication_type == "Communication")
-			& (communication_doctype.communication_medium == "Email")
-			& (communication_doctype.sent_or_received == "Received")
-			& (communication_doctype.email_status.notin(["spam", "Trash"]))
-			& (communication_doctype.email_account.isin(distinct_email_accounts))
-			& (communication_doctype.modified >= Now() - Interval(years=1))
-			& (communication_doctype.seen == 0)
-		)
-	)

frappe/database/database.py

@@ -37,9 +37,9 @@ class Database(object):
 
 	OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"]
 	DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"]
-	STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype')
-	DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent',
-		'parentfield', 'parenttype', 'idx']
+	STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by')
+	DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'idx']
+	CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield')
 	MAX_WRITES_PER_TRANSACTION = 200_000
 
 	class InvalidColumnName(frappe.ValidationError): pass
@@ -435,11 +435,9 @@ class Database(object):
 
 		else:
 			fields = fieldname
-			if fieldname!="*":
+			if fieldname != "*":
 				if isinstance(fieldname, str):
 					fields = [fieldname]
-				else:
-					fields = fieldname
 
 			if (filters is not None) and (filters!=doctype or doctype=="DocType"):
 				try:

frappe/database/mariadb/framework_mariadb.sql

@@ -171,9 +171,6 @@ CREATE TABLE `tabDocType` (
   `modified_by` varchar(255) DEFAULT NULL,
   `owner` varchar(255) DEFAULT NULL,
   `docstatus` int(1) NOT NULL DEFAULT 0,
-  `parent` varchar(255) DEFAULT NULL,
-  `parentfield` varchar(255) DEFAULT NULL,
-  `parenttype` varchar(255) DEFAULT NULL,
   `idx` int(8) NOT NULL DEFAULT 0,
   `search_fields` varchar(255) DEFAULT NULL,
   `issingle` int(1) NOT NULL DEFAULT 0,
@@ -228,8 +225,7 @@ CREATE TABLE `tabDocType` (
   `subject_field` varchar(255) DEFAULT NULL,
   `sender_field` varchar(255) DEFAULT NULL,
   `migration_hash` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`name`),
-  KEY `parent` (`parent`)
+  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --

frappe/database/mariadb/schema.py

@@ -18,6 +18,17 @@ class MariaDBTable(DBTable):
 		if index_defs:
 			additional_definitions += ',\n'.join(index_defs) + ',\n'
 
+		# child table columns
+		if self.meta.get("istable") or 0:
+			additional_definitions += ',\n'.join(
+				(
+					f"parent varchar({varchar_len})",
+					f"parentfield varchar({varchar_len})",
+					f"parenttype varchar({varchar_len})",
+					"index parent(parent)"
+				)
+			) + ',\n'
+
 		# create table
 		query = f"""create table `{self.table_name}` (
 			name varchar({varchar_len}) not null primary key,
@@ -26,12 +37,8 @@ class MariaDBTable(DBTable):
 			modified_by varchar({varchar_len}),
 			owner varchar({varchar_len}),
 			docstatus int(1) not null default '0',
-			parent varchar({varchar_len}),
-			parentfield varchar({varchar_len}),
-			parenttype varchar({varchar_len}),
 			idx int(8) not null default '0',
 			{additional_definitions}
-			index parent(parent),
 			index modified(modified))
 			ENGINE={engine}
 			ROW_FORMAT=DYNAMIC

frappe/database/postgres/framework_postgres.sql

@@ -176,9 +176,6 @@ CREATE TABLE "tabDocType" (
   "modified_by" varchar(255) DEFAULT NULL,
   "owner" varchar(255) DEFAULT NULL,
   "docstatus" smallint NOT NULL DEFAULT 0,
-  "parent" varchar(255) DEFAULT NULL,
-  "parentfield" varchar(255) DEFAULT NULL,
-  "parenttype" varchar(255) DEFAULT NULL,
   "idx" bigint NOT NULL DEFAULT 0,
   "search_fields" varchar(255) DEFAULT NULL,
   "issingle" smallint NOT NULL DEFAULT 0,

frappe/database/postgres/schema.py

@@ -5,26 +5,37 @@ from frappe.database.schema import DBTable, get_definition
 
 class PostgresTable(DBTable):
 	def create(self):
-		add_text = ''
+		add_text = ""
 
 		# columns
 		column_defs = self.get_column_definitions()
-		if column_defs: add_text += ',\n'.join(column_defs)
+		if column_defs:
+			add_text += ",\n".join(column_defs)
+
+		# child table columns
+		if self.meta.get("istable") or 0:
+			if column_defs:
+				add_text += ",\n"
+
+			add_text += ",\n".join(
+				(
+					"parent varchar({varchar_len})",
+					"parentfield varchar({varchar_len})",
+					"parenttype varchar({varchar_len})"
+				)
+			)
 
 		# TODO: set docstatus length
 		# create table
-		frappe.db.sql("""create table `%s` (
+		frappe.db.sql(("""create table `%s` (
 			name varchar({varchar_len}) not null primary key,
 			creation timestamp(6),
 			modified timestamp(6),
 			modified_by varchar({varchar_len}),
 			owner varchar({varchar_len}),
 			docstatus smallint not null default '0',
-			parent varchar({varchar_len}),
-			parentfield varchar({varchar_len}),
-			parenttype varchar({varchar_len}),
 			idx bigint not null default '0',
-			%s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text))
+			%s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN))
 
 		self.create_indexes()
 		frappe.db.commit()

frappe/database/schema.py

@@ -106,6 +106,9 @@ class DBTable:
 
 		columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
 			frappe.db.STANDARD_VARCHAR_COLUMNS]
+		if self.meta.get("istable"):
+			columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
+				frappe.db.CHILD_TABLE_COLUMNS]
 		columns += self.columns.values()
 
 		for col in columns:
@@ -300,11 +303,12 @@ def validate_column_length(fieldname):
 def get_definition(fieldtype, precision=None, length=None):
 	d = frappe.db.type_map.get(fieldtype)
 
-	# convert int to long int if the length of the int is greater than 11
-	if fieldtype == "Int" and length and length > 11:
-		d = frappe.db.type_map.get("Long Int")
+	if not d:
+		return
 
-	if not d: return
+	if fieldtype == "Int" and length and length > 11:
+		# convert int to long int if the length of the int is greater than 11
+		d = frappe.db.type_map.get("Long Int")
 
 	coltype = d[0]
 	size = d[1] if d[1] else None
@@ -315,19 +319,44 @@ def get_definition(fieldtype, precision=None, length=None):
 		if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6:
 			size = '21,9'
 
-		if coltype == "varchar" and length:
-			size = length
+		if length:
+			if coltype == "varchar":
+				size = length
+			elif coltype == "int" and length < 11:
+				# allow setting custom length for int if length provided is less than 11
+				# NOTE: this will only be applicable for mariadb as frappe implements int
+				# in postgres as bigint (as seen in type_map)
+				size = length
 
 	if size is not None:
 		coltype = "{coltype}({size})".format(coltype=coltype, size=size)
 
 	return coltype
 
-def add_column(doctype, column_name, fieldtype, precision=None):
+def add_column(
+	doctype,
+	column_name,
+	fieldtype,
+	precision=None,
+	length=None,
+	default=None,
+	not_null=False
+):
 	if column_name in frappe.db.get_table_columns(doctype):
 		# already exists
 		return
 
 	frappe.db.commit()
-	frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype,
-		column_name, get_definition(fieldtype, precision)))
+
+	query = "alter table `tab%s` add column %s %s" % (
+		doctype,
+		column_name,
+		get_definition(fieldtype, precision, length)
+	)
+
+	if not_null:
+		query += " not null"
+	if default:
+		query += f" default '{default}'"
+
+	frappe.db.sql(query)

frappe/desk/doctype/form_tour/form_tour.js

@@ -15,12 +15,12 @@ frappe.ui.form.on('Form Tour', {
 
 		frm.add_custom_button(__('Show Tour'), async () => {
 			const issingle = await check_if_single(frm.doc.reference_doctype);
-			const name = await get_first_document(frm.doc.reference_doctype);
 			let route_changed = null;
-
+			
 			if (issingle) {
 				route_changed = frappe.set_route('Form', frm.doc.reference_doctype);
 			} else if (frm.doc.first_document) {
+				const name = await get_first_document(frm.doc.reference_doctype);
 				route_changed = frappe.set_route('Form', frm.doc.reference_doctype, name);
 			} else {
 				route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new');
@@ -53,73 +53,69 @@ frappe.ui.form.on('Form Tour', {
 			};
 		});
 
-		frm.set_query("field", "steps", function() {
-			return {
-				query: "frappe.desk.doctype.form_tour.form_tour.get_docfield_list",
-				filters: {
-					doctype: frm.doc.reference_doctype,
-					hidden: 0
-				}
-			};
-		});
-
-		frm.set_query("parent_field", "steps", function() {
-			return {
-				query: "frappe.desk.doctype.form_tour.form_tour.get_docfield_list",
-				filters: {
-					doctype: frm.doc.reference_doctype,
-					fieldtype: "Table",
-					hidden: 0,
-				}
-			};
-		});
-
 		frm.trigger('reference_doctype');
 	},
 
 	reference_doctype(frm) {
 		if (!frm.doc.reference_doctype) return;
 
-		frappe.db.get_list('DocField', {
-			filters: {
-				parent: frm.doc.reference_doctype,
-				parenttype: 'DocType',
-				fieldtype: 'Table'
-			},
-			fields: ['options']
-		}).then(res => {
-			if (Array.isArray(res)) {
-				frm.child_doctypes = res.map(r => r.options);
-			}
+		frm.set_fields_as_options(
+			"fieldname",
+			frm.doc.reference_doctype,
+			df => !df.hidden
+		).then(options => {
+			frm.fields_dict.steps.grid.update_docfield_property(
+				"fieldname",
+				"options",
+				[""].concat(options)
+			);
+		});
+
+		frm.set_fields_as_options(
+			'parent_fieldname',
+			frm.doc.reference_doctype,
+			(df) => df.fieldtype == "Table" && !df.hidden,
+		).then(options => {
+			frm.fields_dict.steps.grid.update_docfield_property(
+				"parent_fieldname",
+				"options",
+				[""].concat(options)
+			);
 		});
 
 	}
 });
 
 frappe.ui.form.on('Form Tour Step', {
-	parent_field(frm, cdt, cdn) {
+	form_render(frm, cdt, cdn) {
+		if (locals[cdt][cdn].is_table_field) {
+			frm.trigger('parent_fieldname', cdt, cdn);
+		}
+	},
+	parent_fieldname(frm, cdt, cdn) {
 		const child_row = locals[cdt][cdn];
-		frappe.model.set_value(cdt, cdn, 'field', '');
-		const field_control = get_child_field("steps", cdn, "field");
-		field_control.get_query = function() {
-			return {
-				query: "frappe.desk.doctype.form_tour.form_tour.get_docfield_list",
-				filters: {
-					doctype: child_row.child_doctype,
-					hidden: 0
-				}
-			};
-		};
+
+		const parent_fieldname_df = frappe
+			.get_meta(frm.doc.reference_doctype)
+			.fields.find(df => df.fieldname == child_row.parent_fieldname);
+
+		frm.set_fields_as_options(
+			'fieldname',
+			parent_fieldname_df.options,
+			(df) => !df.hidden,
+		).then(options => {
+			frm.fields_dict.steps.grid.update_docfield_property(
+				"fieldname",
+				"options",
+				[""].concat(options)
+			);
+			if (child_row.fieldname) {
+				frappe.model.set_value(cdt, cdn, 'fieldname', child_row.fieldname);
+			}
+		});
 	}
 });
 
-function get_child_field(child_table, child_name, fieldname) {
-	// gets the field from grid row form
-	const grid = cur_frm.fields_dict[child_table].grid;
-	const grid_row = grid.grid_rows_by_docname[child_name];
-	return grid_row.grid_form.fields_dict[fieldname];
-}
-
 async function check_if_single(doctype) {
 	const { message } = await frappe.db.get_value('DocType', doctype, 'issingle');
 	return message.issingle || 0;

frappe/desk/doctype/form_tour/form_tour.py

@@ -5,58 +5,23 @@ import frappe
 from frappe.model.document import Document
 from frappe.modules.export_file import export_to_files
 
+
 class FormTour(Document):
-	def before_insert(self):
-		if not self.is_standard:
-			return
+	def before_save(self):
+		meta = frappe.get_meta(self.reference_doctype)
+		for step in self.steps:
+			if step.is_table_field and step.parent_fieldname:
+				parent_field_df = meta.get_field(step.parent_fieldname)
+				step.child_doctype = parent_field_df.options
 
-		# while syncing, set proper docfield reference
-		for d in self.steps:
-			if not frappe.db.exists('DocField', d.field):
-				d.field = frappe.db.get_value('DocField', {
-					'fieldname': d.fieldname, 'parent': self.reference_doctype, 'fieldtype': d.fieldtype
-				}, "name")
-
-			if d.is_table_field and not frappe.db.exists('DocField', d.parent_field):
-				d.parent_field = frappe.db.get_value('DocField', {
-					'fieldname': d.parent_fieldname, 'parent': self.reference_doctype, 'fieldtype': 'Table'
-				}, "name")
+				field_df = frappe.get_meta(step.child_doctype).get_field(step.fieldname)
+				step.label = field_df.label
+				step.fieldtype = field_df.fieldtype
+			else:
+				field_df = meta.get_field(step.fieldname)
+				step.label = field_df.label
+				step.fieldtype = field_df.fieldtype
 
 	def on_update(self):
 		if frappe.conf.developer_mode and self.is_standard:
-			export_to_files([['Form Tour', self.name]], self.module)
-
-	def before_export(self, doc):
-		for d in doc.steps:
-			d.field = ""
-			d.parent_field = ""
-
-@frappe.whitelist()
-@frappe.validate_and_sanitize_search_inputs
-def get_docfield_list(doctype, txt, searchfield, start, page_len, filters):
-	or_filters = [
-		['fieldname', 'like', '%' + txt + '%'],
-		['label', 'like', '%' + txt + '%'],
-		['fieldtype', 'like', '%' + txt + '%']
-	]
-
-	parent_doctype = filters.get('doctype')
-	fieldtype = filters.get('fieldtype')
-	if not fieldtype:
-		excluded_fieldtypes = ['Column Break']
-		excluded_fieldtypes += filters.get('excluded_fieldtypes', [])
-		fieldtype_filter = ['not in', excluded_fieldtypes]
-	else:
-		fieldtype_filter = fieldtype
-
-	docfields = frappe.get_all(
-		doctype,
-		fields=["name as value", "label", "fieldtype"],
-		filters={'parent': parent_doctype, 'fieldtype': fieldtype_filter},
-		or_filters=or_filters,
-		limit_start=start,
-		limit_page_length=page_len,
-		order_by="idx",
-		as_list=1,
-	)
-	return docfields
+			export_to_files([["Form Tour", self.name]], self.module)

frappe/desk/doctype/form_tour_step/form_tour_step.json

@@ -6,19 +6,17 @@
  "field_order": [
   "is_table_field",
   "section_break_2",
-  "parent_field",
-  "field",
+  "parent_fieldname",
+  "fieldname",
   "title",
   "description",
   "column_break_2",
   "position",
   "label",
+  "fieldtype",
   "has_next_condition",
   "next_step_condition",
   "section_break_13",
-  "fieldname",
-  "parent_fieldname",
-  "fieldtype",
   "child_doctype"
  ],
  "fields": [
@@ -38,23 +36,13 @@
    "reqd": 1
   },
   {
-   "depends_on": "eval: (!doc.is_table_field || (doc.is_table_field && doc.parent_field))",
-   "fieldname": "field",
-   "fieldtype": "Link",
-   "label": "Field",
-   "options": "DocField",
+   "depends_on": "eval: (!doc.is_table_field || (doc.is_table_field && doc.parent_fieldname))",
+   "fieldname": "fieldname",
+   "fieldtype": "Select",
+   "label": "Fieldname",
    "reqd": 1
   },
   {
-   "fetch_from": "field.fieldname",
-   "fieldname": "fieldname",
-   "fieldtype": "Data",
-   "hidden": 1,
-   "label": "Fieldname",
-   "read_only": 1
-  },
-  {
-   "fetch_from": "field.label",
    "fieldname": "label",
    "fieldtype": "Data",
    "in_list_view": 1,
@@ -88,10 +76,8 @@
   },
   {
    "default": "0",
-   "fetch_from": "field.fieldtype",
    "fieldname": "fieldtype",
    "fieldtype": "Data",
-   "hidden": 1,
    "label": "Fieldtype",
    "read_only": 1
   },
@@ -105,14 +91,6 @@
    "fieldname": "section_break_2",
    "fieldtype": "Section Break"
   },
-  {
-   "depends_on": "is_table_field",
-   "fieldname": "parent_field",
-   "fieldtype": "Link",
-   "label": "Parent Field",
-   "mandatory_depends_on": "is_table_field",
-   "options": "DocField"
-  },
   {
    "fieldname": "section_break_13",
    "fieldtype": "Section Break",
@@ -120,7 +98,6 @@
    "label": "Hidden Fields"
   },
   {
-   "fetch_from": "parent_field.options",
    "fieldname": "child_doctype",
    "fieldtype": "Data",
    "hidden": 1,
@@ -128,18 +105,17 @@
    "read_only": 1
   },
   {
-   "fetch_from": "parent_field.fieldname",
+   "depends_on": "is_table_field",
    "fieldname": "parent_fieldname",
-   "fieldtype": "Data",
-   "hidden": 1,
-   "label": "Parent Fieldname",
-   "read_only": 1
+   "fieldtype": "Select",
+   "label": "Parent Field",
+   "mandatory_depends_on": "is_table_field"
   }
  ],
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-06-06 20:52:21.076972",
+ "modified": "2022-01-27 15:18:36.481801",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Form Tour Step",
@@ -147,5 +123,6 @@
  "permissions": [],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/desk/doctype/tag/tag.py

@@ -148,8 +148,6 @@ def update_tags(doc, tags):
 			"doctype": "Tag Link",
 			"document_type": doc.doctype,
 			"document_name": doc.name,
-			"parenttype": doc.doctype,
-			"parent": doc.name,
 			"title": doc.get_title() or '',
 			"tag": tag
 		}).insert(ignore_permissions=True)

frappe/desk/doctype/workspace/workspace.json

@@ -20,13 +20,13 @@
   "hide_custom",
   "public",
   "content",
-  "section_break_2",
+  "tab_break_2",
   "charts",
-  "section_break_15",
+  "tab_break_15",
   "shortcuts",
-  "section_break_18",
+  "tab_break_18",
   "links",
-  "roles_section",
+  "roles_tab",
   "roles"
  ],
  "fields": [
@@ -40,8 +40,8 @@
   {
    "collapsible": 1,
    "collapsible_depends_on": "charts",
-   "fieldname": "section_break_2",
-   "fieldtype": "Section Break",
+   "fieldname": "tab_break_2",
+   "fieldtype": "Tab Break",
    "label": "Dashboards"
   },
   {
@@ -78,15 +78,15 @@
   {
    "collapsible": 1,
    "collapsible_depends_on": "shortcuts",
-   "fieldname": "section_break_15",
-   "fieldtype": "Section Break",
+   "fieldname": "tab_break_15",
+   "fieldtype": "Tab Break",
    "label": "Shortcuts"
   },
   {
    "collapsible": 1,
    "collapsible_depends_on": "links",
-   "fieldname": "section_break_18",
-   "fieldtype": "Section Break",
+   "fieldname": "tab_break_18",
+   "fieldtype": "Tab Break",
    "label": "Link Cards"
   },
   {
@@ -152,14 +152,14 @@
    "options": "Has Role"
   },
   {
-   "fieldname": "roles_section",
-   "fieldtype": "Section Break",
+   "fieldname": "roles_tab",
+   "fieldtype": "Tab Break",
    "label": "Roles"
   }
  ],
  "in_create": 1,
  "links": [],
- "modified": "2021-12-15 19:33:00.805265",
+ "modified": "2022-01-27 12:06:13.111743",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace",

frappe/desk/form/linked_with.py

@@ -389,8 +389,6 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 		else:
 			return results
 
-	me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
-
 	for dt, link in linkinfo.items():
 		filters = []
 		link["doctype"] = dt
@@ -413,11 +411,16 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 					ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters"))
 
 				elif link.get("get_parent"):
-					if me and me.parent and me.parenttype == dt:
+					ret = None
+
+					# check for child table
+					if not frappe.get_meta(doctype).istable:
+						continue
+
+					me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
+					if me and me.parenttype == dt:
 						ret = frappe.get_all(doctype=dt, fields=fields,
 							filters=[[dt, "name", '=', me.parent]])
-					else:
-						ret = None
 
 				elif link.get("child_doctype"):
 					or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")]
@@ -473,7 +476,7 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
 	ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled))
 	ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled))
 
-	filters=[['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
+	filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 	# find links of parents
 	links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters)
@@ -498,12 +501,12 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
 
 def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 
-	filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]]
+	filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 
 	# find links of parents
 	links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1)
-	links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
+	links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
 
 	ret = {}
 
@@ -529,34 +532,37 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 	ret = {}
 
-	filters=[['fieldtype','=', 'Dynamic Link']]
+	filters = [['fieldtype','=', 'Dynamic Link']]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 
 	# find dynamic links of parents
 	links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
-	links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
+	links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
 
 	for df in links:
 		if is_single(df.doctype): continue
 
-		# optimized to get both link exists and parenttype
-		possible_link = frappe.get_all(df.doctype, filters={df.doctype_fieldname: doctype},
-		fields=['parenttype'], distinct=True)
+		is_child = frappe.get_meta(df.doctype).istable
+		possible_link = frappe.get_all(
+			df.doctype,
+			filters={df.doctype_fieldname: doctype},
+			fields=["parenttype"] if is_child else None,
+			distinct=True
+		)
 
 		if not possible_link: continue
 
-		for d in possible_link:
-			# is child
-			if d.parenttype:
+		if is_child:
+			for d in possible_link:
 				ret[d.parenttype] = {
 					"child_doctype": df.doctype,
 					"fieldname": [df.fieldname],
 					"doctype_fieldname": df.doctype_fieldname
 				}
-			else:
-				ret[df.doctype] = {
-					"fieldname": [df.fieldname],
-					"doctype_fieldname": df.doctype_fieldname
-				}
+		else:
+			ret[df.doctype] = {
+				"fieldname": [df.fieldname],
+				"doctype_fieldname": df.doctype_fieldname
+			}
 
 	return ret

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -6,7 +6,6 @@ from frappe.utils import strip, cint
 from frappe.translate import (set_default_language, get_dict, send_translations)
 from frappe.geo.country_info import get_country_info
 from frappe.utils.password import update_password
-from werkzeug.useragents import UserAgent
 from . import install_fixtures
 
 def get_setup_stages(args):
@@ -315,17 +314,10 @@ def prettify_args(args):
 	return pretty_args
 
 def email_setup_wizard_exception(traceback, args):
-	if not frappe.local.conf.setup_wizard_exception_email:
+	if not frappe.conf.setup_wizard_exception_email:
 		return
 
 	pretty_args = prettify_args(args)
-
-	if frappe.local.request:
-		user_agent = UserAgent(frappe.local.request.headers.get('User-Agent', ''))
-
-	else:
-		user_agent = frappe._dict()
-
 	message = """
 
 #### Traceback
@@ -349,18 +341,15 @@ def email_setup_wizard_exception(traceback, args):
 #### Basic Information
 
 - **Site:** {site}
-- **User:** {user}
-- **Browser:** {user_agent.platform} {user_agent.browser} version: {user_agent.version} language: {user_agent.language}
-- **Browser Languages**: `{accept_languages}`""".format(
+- **User:** {user}""".format(
 		site=frappe.local.site,
 		traceback=traceback,
 		args="\n".join(pretty_args),
 		user=frappe.session.user,
-		user_agent=user_agent,
-		headers=frappe.local.request.headers,
-		accept_languages=", ".join(frappe.local.request.accept_languages.values()))
+		headers=frappe.request.headers,
+	)
 
-	frappe.sendmail(recipients=frappe.local.conf.setup_wizard_exception_email,
+	frappe.sendmail(recipients=frappe.conf.setup_wizard_exception_email,
 		sender=frappe.session.user,
 		subject="Setup failed: {}".format(frappe.local.site),
 		message=message,

frappe/desk/reportview.py

@@ -6,7 +6,7 @@
 import frappe, json
 import frappe.permissions
 from frappe.model.db_query import DatabaseQuery
-from frappe.model import default_fields, optional_fields
+from frappe.model import default_fields, optional_fields, child_table_fields
 from frappe import _
 from io import StringIO
 from frappe.core.doctype.access_log.access_log import make_access_log
@@ -156,7 +156,7 @@ def raise_invalid_field(fieldname):
 def is_standard(fieldname):
 	if '.' in fieldname:
 		parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None)
-	return fieldname in default_fields or fieldname in optional_fields
+	return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields
 
 def extract_fieldname(field):
 	for text in (',', '/*', '#'):

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -252,7 +252,7 @@ def make_links(columns, data):
 				if col.options and row.get(col.options):
 					row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
 			elif col.fieldtype == "Currency":
-				doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.parent else None
+				doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.get("parent") else None
 				# Pass the Document to get the currency based on docfield option
 				row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc)
 	return columns, data

frappe/email/doctype/email_account/email_account.py

@@ -421,10 +421,10 @@ class EmailAccount(Document):
 	def get_failed_attempts_count(self):
 		return cint(frappe.cache().get('{0}:email-account-failed-attempts'.format(self.name)))
 
-	def receive(self, test_mails=None):
+	def receive(self):
 		"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
 		exceptions = []
-		inbound_mails = self.get_inbound_mails(test_mails=test_mails)
+		inbound_mails = self.get_inbound_mails()
 		for mail in inbound_mails:
 			try:
 				communication = mail.process()
@@ -457,20 +457,19 @@ class EmailAccount(Document):
 		if exceptions:
 			raise Exception(frappe.as_json(exceptions))
 
-	def get_inbound_mails(self, test_mails=None) -> List[InboundMail]:
+	def get_inbound_mails(self) -> List[InboundMail]:
 		"""retrive and return inbound mails.
 
 		"""
 		mails = []
 
-		def process_mail(messages):
+		def process_mail(messages, append_to=None):
 			for index, message in enumerate(messages.get("latest_messages", [])):
 				uid = messages['uid_list'][index] if messages.get('uid_list') else None
-				seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0
-				mails.append(InboundMail(message, self, uid, seen_status))
-
-		if frappe.local.flags.in_test:
-			return [InboundMail(msg, self) for msg in test_mails or []]
+				seen_status = messages.get('seen_status', {}).get(uid)
+				if self.email_sync_option != 'UNSEEN' or seen_status != "SEEN":
+					# only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN'
+					mails.append(InboundMail(message, self, uid, seen_status, append_to))
 
 		if not self.enable_incoming:
 			return []
@@ -481,10 +480,10 @@ class EmailAccount(Document):
 			if self.use_imap:
 				# process all given imap folder
 				for folder in self.imap_folder:
-					email_server.select_imap_folder(folder.folder_name)
-					email_server.settings['uid_validity'] = folder.uidvalidity
-					messages = email_server.get_messages(folder=folder.folder_name) or {}
-					process_mail(messages)
+					if email_server.select_imap_folder(folder.folder_name):
+						email_server.settings['uid_validity'] = folder.uidvalidity
+						messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {}
+						process_mail(messages, folder.append_to)
 			else:
 				# process the pop3 account
 				messages = email_server.get_messages() or {}
@@ -494,7 +493,6 @@ class EmailAccount(Document):
 		except Exception:
 			frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
 			return []
-
 		return mails
 
 	def handle_bad_emails(self, uid, raw, reason):

frappe/email/doctype/email_account/test_email_account.py

@@ -14,11 +14,11 @@ from frappe.core.doctype.communication.email import make
 from frappe.desk.form.load import get_attachments
 from frappe.email.doctype.email_account.email_account import notify_unreplied
 
+from unittest.mock import patch
+
 make_test_records("User")
 make_test_records("Email Account")
 
-
-
 class TestEmailAccount(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
@@ -45,10 +45,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_incoming(self):
 		cleanup("test_sender@example.com")
 
-		test_mails = [self.get_test_mail('incoming-1.raw')]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-1.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("test_receiver@example.com" in comm.recipients)
@@ -72,11 +83,21 @@ class TestEmailAccount(unittest.TestCase):
 		existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'})
 		frappe.delete_doc("File", existing_file.name)
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile:
-			test_mails = [testfile.read()]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-2.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("test_receiver@example.com" in comm.recipients)
@@ -93,11 +114,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_incoming_attached_email_from_outlook_plain_text_only(self):
 		cleanup("test_sender@example.com")
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-3.raw"), "r") as f:
-			test_mails = [f.read()]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-3.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content)
@@ -106,11 +137,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_incoming_attached_email_from_outlook_layers(self):
 		cleanup("test_sender@example.com")
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-4.raw"), "r") as f:
-			test_mails = [f.read()]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-4.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content)
@@ -151,11 +192,23 @@ class TestEmailAccount(unittest.TestCase):
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw"), "r") as f:
 			raw = f.read()
 			raw = raw.replace("<-- in-reply-to -->", sent_mail.get("Message-Id"))
-			test_mails = [raw]
 
 		# parse reply
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					raw
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
+
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		sent = frappe.get_doc("Communication", sent_name)
 
@@ -173,8 +226,20 @@ class TestEmailAccount(unittest.TestCase):
 			test_mails.append(f.read())
 
 		# parse reply
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': test_mails,
+				'seen_status': {
+					2: 'UNSEEN',
+					3: 'UNSEEN'
+				},
+				'uid_list': [2, 3]
+			}
+		}
+
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
 			fields=["name", "reference_doctype", "reference_name"])
@@ -197,11 +262,22 @@ class TestEmailAccount(unittest.TestCase):
 
 		# get test mail with message-id as in-reply-to
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f:
-			test_mails = [f.read().replace('{{ message_id }}', last_mail.message_id)]
+			messages = {
+				# append_to = ToDo
+				'"INBOX"': {
+					'latest_messages': [
+						f.read().replace('{{ message_id }}', last_mail.message_id)
+					],
+					'seen_status': {
+						2: 'UNSEEN'
+					},
+					'uid_list': [2]
+				}
+			}
 
 		# pull the mail
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
 			fields=["name", "reference_doctype", "reference_name"])
@@ -213,10 +289,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_auto_reply(self):
 		cleanup("test_sender@example.com")
 
-		test_mails = [self.get_test_mail('incoming-1.raw')]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-1.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype,
@@ -246,6 +333,91 @@ class TestEmailAccount(unittest.TestCase):
 		with self.assertRaises(Exception):
 			email_account.validate()
 
+	def test_append_to(self):
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		mail_content = self.get_test_mail(fname="incoming-2.raw")
+
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1, 'ToDo')
+		communication = inbound_mail.process()
+		# the append_to for the email is set to ToDO in "_Test Email Account 1"
+		self.assertEqual(communication.reference_doctype, 'ToDo')
+		self.assertTrue(communication.reference_name)
+		self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name))
+
+	def test_append_to_with_imap_folders(self):
+		mail_content_1 = self.get_test_mail(fname="incoming-1.raw")
+		mail_content_2 = self.get_test_mail(fname="incoming-2.raw")
+		mail_content_3 = self.get_test_mail(fname="incoming-3.raw")
+
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					mail_content_1,
+					mail_content_2
+				],
+				'seen_status': {
+					0: 'UNSEEN',
+					1: 'UNSEEN'
+				},
+				'uid_list': [0,1]
+			},
+			# append_to = Communication
+			'"Test Folder"': {
+				'latest_messages': [
+					mail_content_3
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages)
+		self.assertEqual(len(mails), 3)
+
+		inbox_mails = 0
+		test_folder_mails = 0
+
+		for mail in mails:
+			communication = mail.process()
+			if mail.append_to == 'ToDo':
+				inbox_mails += 1
+				self.assertEqual(communication.reference_doctype, 'ToDo')
+				self.assertTrue(communication.reference_name)
+				self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name))
+			else:
+				test_folder_mails += 1
+				self.assertEqual(communication.reference_doctype, None)
+
+		self.assertEqual(inbox_mails, 2)
+		self.assertEqual(test_folder_mails, 1)
+
+	@patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True)
+	@patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None)
+	def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None):
+		from frappe.email.receive import EmailServer
+
+		def get_mocked_messages(**kwargs):
+			return messages.get(kwargs["folder"], {})
+
+		with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages):
+			mails = email_account.get_inbound_mails()
+
+		return mails
+
+	@patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True)
+	@patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None)
+	def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None):
+		def get_mocked_messages(**kwargs):
+			return messages.get(kwargs["folder"], {})
+
+		from frappe.email.receive import EmailServer
+		with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages):
+			email_account.receive()
+
 class TestInboundMail(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
@@ -313,11 +485,11 @@ class TestInboundMail(unittest.TestCase):
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
 		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
-		new_communiction = inbound_mail.process()
+		new_communication = inbound_mail.process()
 
 		# Make sure that uid is changed to new uid
-		self.assertEqual(new_communiction.uid, 12345)
-		self.assertEqual(communication.name, new_communiction.name)
+		self.assertEqual(new_communication.uid, 12345)
+		self.assertEqual(communication.name, new_communication.name)
 
 	def test_find_parent_email_queue(self):
 		"""If the mail is reply to the already sent mail, there will be a email queue record.

frappe/email/doctype/email_account/test_records.json

@@ -20,7 +20,7 @@
 		"pop3_server": "pop.test.example.com",
 		"no_remaining":"0",
 		"append_to": "ToDo",
-		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}],
+		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}],
 		"track_email_status": 1
 	},
 	{

frappe/email/doctype/email_queue/email_queue.py

@@ -479,21 +479,24 @@ class QueueBuilder:
 
 		EmailUnsubscribe = DocType("Email Unsubscribe")
 
-		unsubscribed = (
-			frappe.qb.from_(EmailUnsubscribe).select(
-				EmailUnsubscribe.email
-			).where(
-				EmailUnsubscribe.email.isin(all_ids)
-				& (
-					(
-						(EmailUnsubscribe.reference_doctype == self.reference_doctype)
-						& (EmailUnsubscribe.reference_name == self.reference_name)
-					) | (
-						EmailUnsubscribe.global_unsubscribe == 1
+		if len(all_ids) > 0:
+			unsubscribed = (
+				frappe.qb.from_(EmailUnsubscribe).select(
+					EmailUnsubscribe.email
+				).where(
+					EmailUnsubscribe.email.isin(all_ids)
+					& (
+						(
+							(EmailUnsubscribe.reference_doctype == self.reference_doctype)
+							& (EmailUnsubscribe.reference_name == self.reference_name)
+						) | (
+							EmailUnsubscribe.global_unsubscribe == 1
+						)
 					)
-				)
-			).distinct()
-		).run(pluck=True)
+				).distinct()
+			).run(pluck=True)
+		else:
+			unsubscribed = None
 
 		self._unsubscribed_user_emails = unsubscribed or []
 		return self._unsubscribed_user_emails

frappe/email/receive.py

@@ -108,7 +108,8 @@ class EmailServer:
 				raise
 
 	def select_imap_folder(self, folder):
-		self.imap.select(folder)
+		res = self.imap.select(f'"{folder}"')
+		return res[0] == 'OK'	# The folder exsits TODO: handle other resoponses too
 
 	def logout(self):
 		if cint(self.settings.use_imap):
@@ -582,10 +583,11 @@ class Email:
 class InboundMail(Email):
 	"""Class representation of incoming mail along with mail handlers.
 	"""
-	def __init__(self, content, email_account, uid=None, seen_status=None):
+	def __init__(self, content, email_account, uid=None, seen_status=None, append_to=None):
 		super().__init__(content)
 		self.email_account = email_account
 		self.uid = uid or -1
+		self.append_to = append_to
 		self.seen_status = seen_status or 0
 
 		# System documents related to this mail
@@ -623,15 +625,18 @@ class InboundMail(Email):
 		if self.parent_communication():
 			data['in_reply_to'] = self.parent_communication().name
 
+		append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to
+
 		if self.reference_document():
 			data['reference_doctype'] = self.reference_document().doctype
 			data['reference_name'] = self.reference_document().name
-		elif self.email_account.append_to and self.email_account.append_to != 'Communication':
-			reference_doc = self._create_reference_document(self.email_account.append_to)
-			if reference_doc:
-				data['reference_doctype'] = reference_doc.doctype
-				data['reference_name'] = reference_doc.name
-				data['is_first'] = True
+		else: 
+			if append_to and append_to != 'Communication':
+				reference_doc = self._create_reference_document(append_to)
+				if reference_doc:
+					data['reference_doctype'] = reference_doc.doctype
+					data['reference_name'] = reference_doc.name
+			data['is_first'] = True
 
 		if self.is_notification():
 			# Disable notifications for notification.

frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py

@@ -5,7 +5,7 @@ import frappe
 import json
 from frappe import _
 from frappe.model.document import Document
-from frappe.model import default_fields
+from frappe.model import default_fields, child_table_fields
 
 class DocumentTypeMapping(Document):
 	def validate(self):
@@ -14,7 +14,7 @@ class DocumentTypeMapping(Document):
 	def validate_inner_mapping(self):
 		meta = frappe.get_meta(self.local_doctype)
 		for field_map in self.field_mapping:
-			if field_map.local_fieldname not in default_fields:
+			if field_map.local_fieldname not in (default_fields + child_table_fields):
 				field = meta.get_field(field_map.local_fieldname)
 				if not field:
 					frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx))

frappe/model/__init__.py

@@ -90,11 +90,14 @@ default_fields = (
 	'creation',
 	'modified',
 	'modified_by',
+	'docstatus',
+	'idx'
+)
+
+child_table_fields = (
 	'parent',
 	'parentfield',
-	'parenttype',
-	'idx',
-	'docstatus'
+	'parenttype'
 )
 
 optional_fields = (

frappe/model/base_document.py

@@ -4,7 +4,7 @@
 import frappe
 import datetime
 from frappe import _
-from frappe.model import default_fields, table_fields
+from frappe.model import default_fields, table_fields, child_table_fields
 from frappe.model.naming import set_new_name
 from frappe.model.utils.link_count import notify_link_count
 from frappe.modules import load_doctype_module
@@ -104,6 +104,10 @@ class BaseDocument(object):
 				"balance": 42000
 			})
 		"""
+
+		# QUESTION: why do we need the 1st for loop?
+		# we're essentially setting the values in d, in the 2nd for loop (?)
+
 		# first set default field values of base document
 		for key in default_fields:
 			if key in d:
@@ -208,7 +212,10 @@ class BaseDocument(object):
 			raise ValueError
 
 	def remove(self, doc):
-		self.get(doc.parentfield).remove(doc)
+		# Usage: from the parent doc, pass the child table doc
+		# to remove that child doc from the child table, thus removing it from the parent doc
+		if doc.get("parentfield"):
+			self.get(doc.parentfield).remove(doc)
 
 	def _init_child(self, value, key):
 		if not self.doctype:
@@ -318,12 +325,19 @@ class BaseDocument(object):
 	def docstatus(self, value):
 		self.__dict__["docstatus"] = DocStatus(cint(value))
 
-	def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False):
+	def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False):
 		doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str)
 		doc["doctype"] = self.doctype
 		for df in self.meta.get_table_fields():
 			children = self.get(df.fieldname) or []
-			doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children]
+			doc[df.fieldname] = [
+				d.as_dict(
+					convert_dates_to_str=convert_dates_to_str,
+					no_nulls=no_nulls,
+					no_default_fields=no_default_fields,
+					no_child_table_fields=no_child_table_fields
+				) for d in children
+			]
 
 		if no_nulls:
 			for k in list(doc):
@@ -335,6 +349,11 @@ class BaseDocument(object):
 				if k in default_fields:
 					del doc[k]
 
+		if no_child_table_fields:
+			for k in list(doc):
+				if k in child_table_fields:
+					del doc[k]
+
 		for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"):
 			if self.get(key):
 				doc[key] = self.get(key)
@@ -514,12 +533,12 @@ class BaseDocument(object):
 			if df.fieldtype in table_fields:
 				return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label))
 
-			elif self.parentfield:
+			# check if parentfield exists (only applicable for child table doctype)
+			elif self.get("parentfield"):
 				return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)),
 					_("Row"), self.idx, _("Value missing for"), _(df.label))
 
-			else:
-				return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
+			return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
 
 		missing = []
 
@@ -538,10 +557,11 @@ class BaseDocument(object):
 	def get_invalid_links(self, is_submittable=False):
 		"""Returns list of invalid links and also updates fetch values if not set"""
 		def get_msg(df, docname):
-			if self.parentfield:
+			# check if parentfield exists (only applicable for child table doctype)
+			if self.get("parentfield"):
 				return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname)
-			else:
-				return "{}: {}".format(_(df.label), docname)
+
+			return "{}: {}".format(_(df.label), docname)
 
 		invalid_links = []
 		cancelled_links = []
@@ -615,11 +635,8 @@ class BaseDocument(object):
 		fetch_from_fieldname = df.fetch_from.split('.')[-1]
 		value = values[fetch_from_fieldname]
 		if df.fieldtype in ['Small Text', 'Text', 'Data']:
-			if fetch_from_fieldname in default_fields:
-				from frappe.model.meta import get_default_df
-				fetch_from_df = get_default_df(fetch_from_fieldname)
-			else:
-				fetch_from_df = frappe.get_meta(doctype).get_field(fetch_from_fieldname)
+			from frappe.model.meta import get_default_df
+			fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname)
 
 			if not fetch_from_df:
 				frappe.throw(
@@ -754,9 +771,9 @@ class BaseDocument(object):
 
 
 	def throw_length_exceeded_error(self, df, max_length, value):
-		if self.parentfield and self.idx:
+		# check if parentfield exists (only applicable for child table doctype)
+		if self.get("parentfield"):
 			reference = _("{0}, Row {1}").format(_(self.doctype), self.idx)
-
 		else:
 			reference = "{0} {1}".format(_(self.doctype), self.name)
 
@@ -867,7 +884,7 @@ class BaseDocument(object):
 		:param parentfield: If fieldname is in child table."""
 		from frappe.model.meta import get_field_precision
 
-		if parentfield and not isinstance(parentfield, str):
+		if parentfield and not isinstance(parentfield, str) and parentfield.get("parentfield"):
 			parentfield = parentfield.parentfield
 
 		cache_key = parentfield or "main"
@@ -894,7 +911,7 @@ class BaseDocument(object):
 		from frappe.utils.formatters import format_value
 
 		df = self.meta.get_field(fieldname)
-		if not df and fieldname in default_fields:
+		if not df:
 			from frappe.model.meta import get_default_df
 			df = get_default_df(fieldname)
 

frappe/model/delete_doc.py

@@ -222,32 +222,35 @@ def check_if_doc_is_linked(doc, method="Delete"):
 	"""
 	from frappe.model.rename_doc import get_link_fields
 	link_fields = get_link_fields(doc.doctype)
-	link_fields = [[lf['parent'], lf['fieldname'], lf['issingle']] for lf in link_fields]
+	ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
+
+	for lf in link_fields:
+		link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle']
 
-	for link_dt, link_field, issingle in link_fields:
 		if not issingle:
-			for item in frappe.db.get_values(link_dt, {link_field:doc.name},
-				["name", "parent", "parenttype", "docstatus"], as_dict=True):
-				linked_doctype = item.parenttype if item.parent else link_dt
+			fields = ["name", "docstatus"]
+			if frappe.get_meta(link_dt).istable:
+				fields.extend(["parent", "parenttype"])
 
-				ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
+			for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True):
+				# available only in child table cases
+				item_parent = getattr(item, "parent", None)
+				linked_doctype = item.parenttype if item_parent else link_dt
 
 				if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'):
 					# don't check for communication and todo!
 					continue
 
-				if not item:
-					continue
-				elif method != "Delete"  and (method != "Cancel" or item.docstatus != 1):
+				if method != "Delete" and (method != "Cancel" or item.docstatus != 1):
 					# don't raise exception if not
 					# linked to a non-cancelled doc when deleting or to a submitted doc when cancelling
 					continue
-				elif link_dt == doc.doctype and (item.parent or item.name) == doc.name:
+				elif link_dt == doc.doctype and (item_parent or item.name) == doc.name:
 					# don't raise exception if not
 					# linked to same item or doc having same name as the item
 					continue
 				else:
-					reference_docname = item.parent or item.name
+					reference_docname = item_parent or item.name
 					raise_link_exists_exception(doc, linked_doctype, reference_docname)
 
 		else:

frappe/model/document.py

@@ -9,7 +9,7 @@ import frappe
 from frappe import _, msgprint, is_whitelisted
 from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff
 from frappe.model.base_document import BaseDocument, get_controller
-from frappe.model.naming import set_new_name
+from frappe.model.naming import set_new_name, validate_name
 from frappe.model.docstatus import DocStatus
 from frappe.model import optional_fields, table_fields
 from frappe.model.workflow import validate_workflow
@@ -416,12 +416,12 @@ class Document(BaseDocument):
 
 		# If autoname has set as Prompt (name)
 		if self.get("__newname"):
-			self.name = self.get("__newname")
+			self.name = validate_name(self.doctype, self.get("__newname"))
 			self.flags.name_set = True
 			return
 
 		if set_name:
-			self.name = set_name
+			self.name = validate_name(self.doctype, set_name)
 		else:
 			set_new_name(self)
 
@@ -527,7 +527,7 @@ class Document(BaseDocument):
 
 	def _validate_non_negative(self):
 		def get_msg(df):
-			if self.parentfield:
+			if self.get("parentfield"):
 				return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)),
 					_("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label)))
 			else:
@@ -1202,7 +1202,7 @@ class Document(BaseDocument):
 		if not frappe.compare(val1, condition, val2):
 			label = doc.meta.get_label(fieldname)
 			condition_str = error_condition_map.get(condition, condition)
-			if doc.parentfield:
+			if doc.get("parentfield"):
 				msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2)
 			else:
 				msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2)
@@ -1226,7 +1226,7 @@ class Document(BaseDocument):
 				doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]}))
 
 		for fieldname in fieldnames:
-			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield)))
+			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield"))))
 
 	def get_url(self):
 		"""Returns Desk URL for this document."""
@@ -1379,9 +1379,11 @@ class Document(BaseDocument):
 		doctype = self.__class__.__name__
 
 		docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
-		parent = f" parent={self.parent}" if self.parent else ""
+		repr_str = f"<{doctype}: {name}{docstatus}"
 
-		return f"<{doctype}: {name}{docstatus}{parent}>"
+		if not hasattr(self, "parent"):
+			return repr_str + ">"
+		return f"{repr_str} parent={self.parent}>"
 
 	def __str__(self):
 		name = self.name or "unsaved"

frappe/model/mapper.py

@@ -4,7 +4,7 @@ import json
 
 import frappe
 from frappe import _
-from frappe.model import default_fields, table_fields
+from frappe.model import default_fields, table_fields, child_table_fields
 from frappe.utils import cstr
 
 
@@ -149,6 +149,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent):
 	no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
 		+ [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
 		+ list(default_fields)
+		+ list(child_table_fields)
 		+ list(table_map.get("field_no_map", [])))
 
 	for df in target_doc.meta.get("fields"):

frappe/model/meta.py

@@ -18,7 +18,7 @@ from datetime import datetime
 import click
 import frappe, json, os
 from frappe.utils import cstr, cint, cast
-from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields
+from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields, child_table_fields
 from frappe.model.document import Document
 from frappe.model.base_document import BaseDocument
 from frappe.modules import load_doctype_module
@@ -191,6 +191,8 @@ class Meta(Document):
 			else:
 				self._valid_columns = self.default_fields + \
 					[df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes]
+				if self.istable:
+					self._valid_columns += list(child_table_fields)
 
 		return self._valid_columns
 
@@ -520,7 +522,7 @@ class Meta(Document):
 		'''add `links` child table in standard link dashboard format'''
 		dashboard_links = []
 
-		if hasattr(self, 'links') and self.links:
+		if getattr(self, 'links', None):
 			dashboard_links.extend(self.links)
 
 		if not data.transactions:
@@ -625,9 +627,9 @@ def get_field_currency(df, doc=None):
 		frappe.local.field_currency = frappe._dict()
 
 	if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or
-		(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
+		(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
 
-		ref_docname = doc.parent or doc.name
+		ref_docname = doc.get("parent") or doc.name
 
 		if ":" in cstr(df.get("options")):
 			split_opts = df.get("options").split(":")
@@ -635,7 +637,7 @@ def get_field_currency(df, doc=None):
 				currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2])
 		else:
 			currency = doc.get(df.get("options"))
-			if doc.parent:
+			if doc.get("parenttype"):
 				if currency:
 					ref_docname = doc.name
 				else:
@@ -648,7 +650,7 @@ def get_field_currency(df, doc=None):
 				.setdefault(df.fieldname, currency)
 
 	return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \
-		(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
+		(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
 
 def get_field_precision(df, doc=None, currency=None):
 	"""get precision based on DocField options and fieldvalue in doc"""
@@ -669,19 +671,25 @@ def get_field_precision(df, doc=None, currency=None):
 
 
 def get_default_df(fieldname):
-	if fieldname in default_fields:
+	if fieldname in (default_fields + child_table_fields):
 		if fieldname in ("creation", "modified"):
 			return frappe._dict(
 				fieldname = fieldname,
 				fieldtype = "Datetime"
 			)
 
-		else:
+		elif fieldname in ("idx", "docstatus"):
 			return frappe._dict(
 				fieldname = fieldname,
-				fieldtype = "Data"
+				fieldtype = "Int"
 			)
 
+		return frappe._dict(
+			fieldname = fieldname,
+			fieldtype = "Data"
+		)
+
+
 def trim_tables(doctype=None, dry_run=False, quiet=False):
 	"""
 	Removes database fields that don't exist in the doctype (json or custom field). This may be needed
@@ -713,7 +721,7 @@ def trim_tables(doctype=None, dry_run=False, quiet=False):
 
 def trim_table(doctype, dry_run=True):
 	frappe.cache().hdel('table_columns', f"tab{doctype}")
-	ignore_fields = default_fields + optional_fields
+	ignore_fields = default_fields + optional_fields + child_table_fields
 	columns = frappe.db.get_table_columns(doctype)
 	fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value()
 	is_internal = lambda f: f not in ignore_fields and not f.startswith("_")

frappe/modules/export_file.py

@@ -47,7 +47,7 @@ def strip_default_fields(doc, doc_export):
 
 	for df in doc.meta.get_table_fields():
 		for d in doc_export.get(df.fieldname):
-			for fieldname in frappe.model.default_fields:
+			for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
 				if fieldname in d:
 					del d[fieldname]
 

frappe/patches.txt

@@ -119,7 +119,6 @@ execute:frappe.delete_doc_if_exists('DocType', 'GSuite Settings')
 execute:frappe.delete_doc_if_exists('DocType', 'GSuite Templates')
 execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Account')
 execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings')
-frappe.patches.v12_0.remove_parent_and_parenttype_from_print_formats
 frappe.patches.v12_0.remove_example_email_thread_notify
 execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders()
 frappe.patches.v12_0.set_correct_url_in_files

frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py

@@ -1,14 +0,0 @@
-import frappe
-
-def execute():
-    frappe.db.sql("""
-        UPDATE
-            `tabPrint Format`
-        SET
-            `tabPrint Format`.`parent`='',
-            `tabPrint Format`.`parenttype`='',
-            `tabPrint Format`.parentfield=''
-        WHERE
-            `tabPrint Format`.parent != ''
-            OR `tabPrint Format`.parenttype != ''
-        """)

frappe/public/js/frappe/form/dashboard.js

@@ -549,14 +549,14 @@ frappe.ui.form.Dashboard = class FormDashboard {
 	render_graph(args) {
 		this.chart_area.show();
 		this.chart_area.body.empty();
-		$.extend({
+		$.extend(args, {
 			type: 'line',
 			colors: ['green'],
 			truncateLegends: 1,
 			axisOptions: {
 				shortenYAxisNumbers: 1
 			}
-		}, args);
+		});
 		this.show();
 
 		this.chart = new frappe.Chart('.form-graph', args);

frappe/public/js/frappe/form/form_tour.js

@@ -151,7 +151,7 @@ frappe.ui.form.FormTour = class FormTour {
 
 			const curr_step = step_info;
 			const next_step = this.tour.steps[curr_step.idx];
-			const is_next_field_in_curr_table = next_step.parent_field == curr_step.field;
+			const is_next_field_in_curr_table = next_step.parent_fieldname == curr_step.fieldname;
 
 			if (!is_next_field_in_curr_table) return;
 

frappe/public/js/frappe/model/model.js

@@ -10,8 +10,7 @@ $.extend(frappe.model, {
 	layout_fields: ['Section Break', 'Column Break', 'Tab Break', 'Fold'],
 
 	std_fields_list: ['name', 'owner', 'creation', 'modified', 'modified_by',
-		'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus',
-		'parent', 'parenttype', 'parentfield', 'idx'],
+		'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus', 'idx'],
 
 	core_doctypes_list: ['DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role',
 		'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form',

frappe/public/js/frappe/ui/messages.js

@@ -233,7 +233,7 @@ frappe.msgprint = function(msg, title, is_minimizable) {
 	if(data.title || !msg_exists) {
 		// set title only if it is explicitly given
 		// and no existing title exists
-		frappe.msg_dialog.set_title(data.title || __('Message'));
+		frappe.msg_dialog.set_title(data.title || __('Message', null, 'Default title of the message dialog'));
 	}
 
 	// show / hide indicator

frappe/public/js/frappe/ui/notifications/notifications.js

@@ -283,12 +283,13 @@ class NotificationsView extends BaseNotificationsView {
 				e.stopImmediatePropagation();
 				this.mark_as_read(field.name, item_html);
 			});
-
-			item_html.on('click', () => {
-				this.mark_as_read(field.name, item_html);
-			});
 		}
 
+		item_html.on('click', () => {
+			!field.read && this.mark_as_read(field.name, item_html);
+			this.notifications_icon.trigger('click');
+		});
+
 		return item_html;
 	}
 

frappe/public/js/frappe/views/workspace/blocks/paragraph.js

@@ -62,7 +62,7 @@ export default class Paragraph extends Block {
 				this.show_hide_block_list();
 			});
 			div.addEventListener('blur', () => {
-				setTimeout(() => this.show_hide_block_list(true), 10);
+				!this.over_block_list_item && this.show_hide_block_list(true);
 			});
 			div.dataset.placeholder = this.api.i18n.t(this._placeholder);
 			div.addEventListener('keyup', this.onKeyUp);
@@ -95,6 +95,12 @@ export default class Paragraph extends Block {
 				this.api.caret.setToBlock(index);
 			});
 
+			$block_list_item.mouseenter(() => {
+				this.over_block_list_item = true;
+			}).mouseleave(() => {
+				this.over_block_list_item = false;
+			});
+
 			$block_list_container.append($block_list_item);
 		});
 

frappe/public/js/frappe/views/workspace/workspace.js

@@ -376,7 +376,7 @@ frappe.views.Workspace = class Workspace {
 		this.clear_page_actions();
 
 		page.is_editable && this.page.set_primary_action(
-			__("Save Customizations"),
+			__("Save"),
 			() => {
 				this.clear_page_actions();
 				this.save_page(page).then((saved) => {
@@ -1158,7 +1158,7 @@ frappe.views.Workspace = class Workspace {
 					item.data.card_name !== 'Custom Reports')
 			);
 
-			if (page.content == JSON.stringify(blocks)) {
+			if (page.content == JSON.stringify(blocks) && Object.keys(new_widgets).length === 0) {
 				this.setup_customization_buttons(page);
 				frappe.show_alert({ message: __("No changes made on the page"), indicator: "warning" });
 				return false;

frappe/public/js/frappe/widgets/base_widget.js

@@ -100,7 +100,7 @@ export default class Widget {
 		let title = max_chars ? frappe.ellipsis(base, max_chars) : base;
 
 		if (this.icon) {
-			let icon = frappe.utils.icon(this.icon);
+			let icon = frappe.utils.icon(this.icon, "lg");
 			this.title_field[0].innerHTML = `${icon} <span class="ellipsis" title="${title}">${title}</span>`;
 		} else {
 			this.title_field[0].innerHTML = `<span class="ellipsis" title="${title}">${title}</span>`;

frappe/public/js/frappe/widgets/widget_dialog.js

@@ -154,7 +154,7 @@ class CardDialog extends WidgetDialog {
 			{
 				fieldtype: "Data",
 				fieldname: "label",
-				label: "Label",
+				label: "Label"
 			},
 			{
 				fieldname: 'links',
@@ -174,7 +174,7 @@ class CardDialog extends WidgetDialog {
 					},
 					{
 						fieldname: "icon",
-						fieldtype: "Data",
+						fieldtype: "Icon",
 						label: "Icon"
 					},
 					{
@@ -182,6 +182,7 @@ class CardDialog extends WidgetDialog {
 						fieldtype: "Select",
 						in_list_view: 1,
 						label: "Link Type",
+						reqd: 1,
 						options: ["DocType", "Page", "Report"]
 					},
 					{
@@ -189,9 +190,9 @@ class CardDialog extends WidgetDialog {
 						fieldtype: "Dynamic Link",
 						in_list_view: 1,
 						label: "Link To",
+						reqd: 1,
 						get_options: (df) => {
 							return df.doc.link_type;
-
 						}
 					},
 					{
@@ -227,6 +228,31 @@ class CardDialog extends WidgetDialog {
 	}
 
 	process_data(data) {
+		data.links.map((item, idx) => {
+			let message = '';
+			let row = idx+1;
+
+			if (!item.link_type) {
+				message = "Following fields have missing values: <br><br><ul>";
+				message += `<li>Link Type in Row ${row}</li>`;
+			}
+
+			if (!item.link_to) {
+				message += `<li>Link To in Row ${row}</li>`;
+			}
+
+			if (message) {
+				message += "</ul>";
+				frappe.throw({
+					message: __(message),
+					title: __("Missing Values Required"),
+					indicator: 'orange'
+				});
+			}
+
+			item.label = item.label ? item.label : item.link_to;
+		});
+
 		data.label = data.label ? data.label : data.chart_name;
 		return data;
 	}

frappe/public/scss/desk/desktop.scss

@@ -155,6 +155,7 @@ body {
 				svg {
 					flex: none;
 					margin-right: 6px;
+					margin-left: -2px;
 					box-shadow: none;
 				}
 			}
@@ -560,21 +561,29 @@ body {
 	}
 
 	&.links-widget-box {
+		padding: 18px 12px;
+
 		.link-item {
 			display: flex;
 			text-decoration: none;
+			font-size: var(--text-md);
 			color: var(--text-color);
-			padding: var(--padding-xs);
-			margin-left: -5px;
+			padding: 4px;
+			margin-left: -4px;
+			margin-bottom: 4px;
 			border-radius: var(--border-radius-md);
 			cursor: pointer;
 
 			&:hover {
-				background-color: var(--bg-color);
+				background-color: var(--fg-hover-color);
+
+				.indicator-pill {
+					background-color: var(--fg-color);
+				}
 			}
 
 			&:first-child {
-				margin-top: 15px;
+				margin-top: 18px;
 			}
 
 			&:last-child {
@@ -601,6 +610,8 @@ body {
 
 			.indicator-pill {
 				margin-right: var(--margin-sm);
+				height: 20px;
+				padding: 3px 8px;
 			}
 		}
 	}
@@ -850,10 +861,16 @@ body {
 		}
 	}
 
+	.layout-main-section-wrapper {
+		margin-top: -5px;
+		padding-top: 5px;
+	}
+
 	.layout-main-section {
 		background-color: var(--fg-color);
-		padding: var(--padding-sm);
+		box-shadow: var(--card-shadow);
 		border-radius: var(--border-radius-lg);
+		padding: var(--padding-sm);
 	}
 
 	.block-menu-item-icon svg{

frappe/query_builder/utils.py

@@ -59,10 +59,28 @@ def patch_query_execute():
 		return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep
 
 	def prepare_query(query):
+		import inspect
+
 		param_collector = NamedParameterWrapper()
 		query = query.get_sql(param_wrapper=param_collector)
 		if frappe.flags.in_safe_exec and not query.lower().strip().startswith("select"):
-			raise frappe.PermissionError('Only SELECT SQL allowed in scripting')
+			callstack = inspect.stack()
+			if len(callstack) >= 3 and ".py" in callstack[2].filename:
+				# ignore any query builder methods called from python files
+				# assumption is that those functions are whitelisted already.
+
+				# since query objects are patched everywhere any query.run()
+				# will have callstack like this:
+				# frame0: this function prepare_query()
+				# frame1: execute_query()
+				# frame2: frame that called `query.run()`
+				#
+				# if frame2 is server script it wont have a filename and hence
+				# it shouldn't be allowed.
+				# ps. stack() returns `"<unknown>"` as filename.
+				pass
+			else:
+				raise frappe.PermissionError('Only SELECT SQL allowed in scripting')
 		return query, param_collector.get_parameters()
 
 	query_class = get_attr(str(frappe.qb).split("'")[1])

frappe/search/full_text_search.py

@@ -66,7 +66,7 @@ class FullTextSearch:
 
 		ix = self.get_index()
 		with ix.searcher():
-			writer = ix.writer()
+			writer = AsyncWriter(ix)
 			writer.delete_by_term(self.id, doc_name)
 			writer.commit(optimize=True)
 
@@ -98,7 +98,7 @@ class FullTextSearch:
 	def build_index(self):
 		"""Build index for all parsed documents"""
 		ix = self.create_index()
-		writer = ix.writer()
+		writer = AsyncWriter(ix)
 
 		for i, document in enumerate(self.documents):
 			if document:

frappe/templates/includes/comments/comments.py

@@ -6,6 +6,7 @@ from frappe.website.utils import clear_cache
 from frappe.rate_limiter import rate_limit
 from frappe.utils import add_to_date, now
 from frappe.website.doctype.blog_settings.blog_settings import get_comment_limit
+from frappe.utils.html_utils import clean_html
 
 from frappe import _
 
@@ -29,7 +30,7 @@ def add_comment(comment, comment_email, comment_by, reference_doctype, reference
 		return False
 
 	comment = doc.add_comment(
-		text=comment,
+		text=clean_html(comment),
 		comment_email=comment_email,
 		comment_by=comment_by)
 

frappe/tests/test_child_table.py

@@ -0,0 +1,66 @@
+import frappe
+from frappe.model import child_table_fields
+
+import unittest
+from typing import Callable
+
+
+class TestChildTable(unittest.TestCase):
+	def tearDown(self) -> None:
+		try:
+			frappe.delete_doc("DocType", self.doctype_name, force=1)
+		except Exception:
+			pass
+
+	def test_child_table_doctype_creation_and_transitioning(self) -> None:
+		'''
+		This method tests the creation of child table doctype
+		as well as it's transitioning from child table to normal and normal to child table doctype
+		'''
+
+		self.doctype_name = "Test Newy Child Table"
+
+		try:
+			doc = frappe.get_doc({
+				"doctype": "DocType",
+				"name": self.doctype_name,
+				"istable": 1,
+				"custom": 1,
+				"module": "Integrations",
+				"fields": [{
+					"label": "Some Field",
+					"fieldname": "some_fieldname",
+					"fieldtype": "Data",
+					"reqd": 1
+				}]
+			}).insert(ignore_permissions=True)
+		except Exception:
+			self.fail("Not able to create Child Table Doctype")
+
+
+		for column in child_table_fields:
+			self.assertTrue(frappe.db.has_column(self.doctype_name, column))
+
+		# check transitioning from child table to normal doctype
+		doc.istable = 0
+		try:
+			doc.save(ignore_permissions=True)
+		except Exception:
+			self.fail("Not able to transition from Child Table Doctype to Normal Doctype")
+
+		self.check_valid_columns(self.assertFalse)
+
+		# check transitioning from normal to child table doctype
+		doc.istable = 1
+		try:
+			doc.save(ignore_permissions=True)
+		except Exception:
+			self.fail("Not able to transition from Normal Doctype to Child Table Doctype")
+
+		self.check_valid_columns(self.assertTrue)
+
+
+	def check_valid_columns(self, assertion_method: Callable) -> None:
+		valid_columns = frappe.get_meta(self.doctype_name).get_valid_columns()
+		for column in child_table_fields:
+			assertion_method(column in valid_columns)

frappe/tests/test_db_update.py

@@ -103,10 +103,7 @@ def get_other_fields_meta(meta):
 	default_fields_map = {
 		'name': ('Data', 0),
 		'owner': ('Data', 0),
-		'parent': ('Data', 0),
-		'parentfield': ('Data', 0),
 		'modified_by': ('Data', 0),
-		'parenttype': ('Data', 0),
 		'creation': ('Datetime', 0),
 		'modified': ('Datetime', 0),
 		'idx': ('Int', 8),
@@ -117,8 +114,12 @@ def get_other_fields_meta(meta):
 	if meta.track_seen:
 		optional_fields.append('_seen')
 
+	child_table_fields_map = {}
+	if meta.istable:
+		child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS})
+
 	optional_fields_map = {field: ('Text', 0) for field in optional_fields}
-	fields = dict(default_fields_map, **optional_fields_map)
+	fields = dict(default_fields_map, **optional_fields_map, **child_table_fields_map)
 	field_map =  [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()]
 
 	return field_map

frappe/tests/test_email.py

@@ -3,6 +3,8 @@
 
 import unittest, frappe, re, email
 
+from frappe.email.doctype.email_account.test_email_account import TestEmailAccount 
+
 test_dependencies = ['Email Account']
 
 class TestEmail(unittest.TestCase):
@@ -173,12 +175,35 @@ class TestEmail(unittest.TestCase):
 		frappe.db.delete("Communication", {"sender": "sukh@yyy.com"})
 
 		with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw:
-			mails = email_account.get_inbound_mails(test_mails=[raw.read()])
+			messages = {
+				# append_to = ToDo
+				'"INBOX"': {
+					'latest_messages': [
+						raw.read()
+					],
+					'seen_status': {
+						2: 'UNSEEN'
+					},
+					'uid_list': [2]
+				}	
+			}
+
+			email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+			changed_flag = False
+			if not email_account.enable_incoming: 
+				email_account.enable_incoming = True
+				changed_flag = True
+			mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages)
+
+			# mails = email_account.get_inbound_mails(test_mails=[raw.read()])
 			communication = mails[0].process()
 
 		self.assertTrue(re.search('''<img[^>]*src=["']/private/files/rtco1.png[^>]*>''', communication.content))
 		self.assertTrue(re.search('''<img[^>]*src=["']/private/files/rtco2.png[^>]*>''', communication.content))
 
+		if changed_flag:
+			email_account.enable_incoming = False
+
 
 if __name__ == '__main__':
 	frappe.connect()

frappe/tests/test_naming.py

@@ -10,11 +10,11 @@ from frappe.model.naming import append_number_if_name_exists, revert_series_if_l
 from frappe.model.naming import determine_consecutive_week_number, parse_naming_series
 
 class TestNaming(unittest.TestCase):
+	def setUp(self):
+		frappe.db.delete('Note')
+
 	def tearDown(self):
-		# Reset ToDo autoname to hash
-		todo_doctype = frappe.get_doc('DocType', 'ToDo')
-		todo_doctype.autoname = 'hash'
-		todo_doctype.save()
+		frappe.db.rollback()
 
 	def test_append_number_if_name_exists(self):
 		'''
@@ -203,4 +203,51 @@ class TestNaming(unittest.TestCase):
 
 		dt = datetime.fromisoformat("2021-12-31")
 		w = determine_consecutive_week_number(dt)
-		self.assertEqual(w, "52")
+		self.assertEqual(w, "52")
+
+	def test_naming_validations(self):
+		# case 1: check same name as doctype
+		# set name via prompt
+		tag = frappe.get_doc({
+			'doctype': 'Tag',
+			'__newname': 'Tag'
+		})
+		self.assertRaises(frappe.NameError, tag.insert)
+
+		# set by passing set_name as ToDo
+		self.assertRaises(frappe.NameError, make_invalid_todo)
+
+		# set new name - Note
+		note = frappe.get_doc({
+			'doctype': 'Note',
+			'title': 'Note'
+		})
+		self.assertRaises(frappe.NameError, note.insert)
+
+		# case 2: set name with "New ---"
+		tag = frappe.get_doc({
+			'doctype': 'Tag',
+			'__newname': 'New Tag'
+		})
+		self.assertRaises(frappe.NameError, tag.insert)
+
+		# case 3: set name with special characters
+		tag = frappe.get_doc({
+			'doctype': 'Tag',
+			'__newname': 'Tag<>'
+		})
+		self.assertRaises(frappe.NameError, tag.insert)
+
+		# case 4: no name specified
+		tag = frappe.get_doc({
+			'doctype': 'Tag',
+			'__newname': ''
+		})
+		self.assertRaises(frappe.ValidationError, tag.insert)
+
+
+def make_invalid_todo():
+	frappe.get_doc({
+		'doctype': 'ToDo',
+		'description': 'Test'
+	}).insert(set_name='ToDo')

frappe/tests/ui_test_helpers.py

@@ -148,9 +148,6 @@ def create_form_tour():
 	if frappe.db.exists('Form Tour', {'name': 'Test Form Tour'}):
 		return
 
-	def get_docfield_name(filters):
-		return frappe.db.get_value('DocField', filters, "name")
-
 	tour = frappe.get_doc({
 		'doctype': 'Form Tour',
 		'title': 'Test Form Tour',
@@ -161,7 +158,6 @@ def create_form_tour():
 			"description": "Test Description 1",
 			"has_next_condition": 1,
 			"next_step_condition": "eval: doc.first_name",
-			"field": get_docfield_name({'parent': 'Contact', 'fieldname': 'first_name'}),
 			"fieldname": "first_name",
 			"fieldtype": "Data"
 		},{
@@ -169,21 +165,18 @@ def create_form_tour():
 			"description": "Test Description 2",
 			"has_next_condition": 1,
 			"next_step_condition": "eval: doc.last_name",
-			"field": get_docfield_name({'parent': 'Contact', 'fieldname': 'last_name'}),
 			"fieldname": "last_name",
 			"fieldtype": "Data"
 		},{
 			"title": "Test Title 3",
 			"description": "Test Description 3",
-			"field": get_docfield_name({'parent': 'Contact', 'fieldname': 'phone_nos'}),
 			"fieldname": "phone_nos",
 			"fieldtype": "Table"
 		},{
 			"title": "Test Title 4",
 			"description": "Test Description 4",
 			"is_table_field": 1,
-			"parent_field": get_docfield_name({'parent': 'Contact', 'fieldname': 'phone_nos'}),
-			"field": get_docfield_name({'parent': 'Contact Phone', 'fieldname': 'phone'}),
+			"parent_fieldname": "phone_nos",
 			"next_step_condition": "eval: doc.phone",
 			"has_next_condition": 1,
 			"fieldname": "phone",

frappe/translations/de.csv

@@ -151,6 +151,7 @@ My Account,Mein Konto,
 New Address,Neue Adresse,
 New Contact,Neuer Kontakt,
 Next,Weiter,
+No,Nein,
 No Data,Keine Daten,
 No address added yet.,Noch keine Adresse hinzugefügt.,
 No contacts added yet.,Noch keine Kontakte hinzugefügt.,
@@ -349,7 +350,7 @@ Add a New Role,Neue Rolle hinzufügen,
 Add a column,Spalte einfügen,
 Add a comment,Einen Kommentar hinzufügen,
 Add a new section,Fügen Sie einen neuen Abschnitt hinzu,
-Add a tag ...,Füge einen Tag hinzu ...,
+Add a tag ...,Füge ein Schlagwort hinzu ...,
 Add all roles,Alle Rollen hinzufügen,
 Add custom forms.,Benutzerdefinierte Formulare hinzufügen,
 Add custom javascript to forms.,Benutzerdefiniertes Javascript zum Formular hinzufügen,
@@ -946,6 +947,7 @@ Edit Auto Email Report Settings,Bearbeiten Sie die Einstellungen für automatisc
 Edit Custom HTML,Benutzerdefiniertes HTML bearbeiten,
 Edit DocType,DocType bearbeiten,
 Edit Filter,Filter bearbeiten,
+Edit Filters,Filter bearbeiten,
 Edit Format,Format bearbeiten,
 Edit HTML,HTML bearbeiten,
 Edit Heading,Kopf bearbeiten,
@@ -1230,6 +1232,7 @@ Hide Copy,Kopie ausblenden,
 Hide Footer Signup,Fußzeilen-Anmeldung ausblenden,
 Hide Sidebar and Menu,Seitenleiste und Menü ausblenden,
 Hide Standard Menu,Standardmenü ausblenden,
+Hide Tags,Schlagworte ausblenden,
 Hide Weekends,Wochenenden ausblenden,
 Hide details,Details ausblenden,
 Hide footer in auto email reports,Fußzeile in automatischen E-Mail-Berichten ausblenden,
@@ -1650,7 +1653,7 @@ No Preview,Keine Vorschau,
 No Preview Available,Keine Vorschau vorhanden,
 No Printer is Available.,Es ist kein Drucker verfügbar.,
 No Results,Keine Ergebnisse,
-No Tags,No Tags,
+No Tags,Keine Schlagworte,
 No alerts for today,Keine Warnungen für heute,
 No comments yet,Noch keine Kommentare,
 No comments yet. Start a new discussion.,Noch keine Kommentare. Starten Sie eine neue Diskussion.,
@@ -2040,7 +2043,7 @@ Remove,Entfernen,
 Remove Field,Feld entfernen,
 Remove Filter,Filter entfernen,
 Remove Section,Abschnitt entfernen,
-Remove Tag,Markierung entfernen,
+Remove Tag,Schlagwort entfernen,
 Remove all customizations?,Alle Anpassungen entfernen?,
 Removed {0},{0} entfernt,
 Rename many items by uploading a .csv file.,Viele Elemente auf einmal umbenennen durch Hochladen einer .CSV-Datei,
@@ -3250,7 +3253,7 @@ DocType Action,DocType-Aktion,
 DocType Event,DocType-Ereignis,
 DocType Link,DocType Link,
 Document Share,Dokumentenfreigabe,
-Document Tag,Dokument-Tag,
+Document Tag,Dokument-Schlagwort,
 Document Title,Dokumenttitel,
 Document Type Field Mapping,Dokumenttyp-Feldzuordnung,
 Document Type Mapping,Dokumenttypzuordnung,
@@ -3796,7 +3799,7 @@ Start,Start,
 Start Time,Startzeit,
 Status,Status,
 Submitted,Gebucht,
-Tag,Etikett,
+Tag,Schlagwort,
 Template,Vorlage,
 Thursday,Donnerstag,
 Title,Bezeichnung,
@@ -4028,7 +4031,7 @@ Please select target language for translation,Bitte wählen Sie die Zielsprache
 Select Language,Sprache auswählen,
 Confirm Translations,Übersetzungen bestätigen,
 Contributed Translations,Beigetragene Übersetzungen,
-Show Tags,Tags anzeigen,
+Show Tags,Schlagworte anzeigen,
 Do not have permission to access {0} bucket.,Sie haben keine Berechtigung zum Zugriff auf den Bucket {0}.,
 Allow document creation via Email,Dokumenterstellung per E-Mail zulassen,
 Sender Field,Absenderfeld,

frappe/utils/__init__.py

@@ -24,9 +24,6 @@ import frappe
 from frappe.utils.data import *
 from frappe.utils.html_utils import sanitize_html
 
-default_fields = ['doctype', 'name', 'owner', 'creation', 'modified', 'modified_by',
-	'parent', 'parentfield', 'parenttype', 'idx', 'docstatus']
-
 
 def get_fullname(user=None):
 	"""get the full name (first name + last name) of the user from User"""

frappe/utils/data.py

@@ -1362,7 +1362,7 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) -
 			"fieldtype":
 		}
 	"""
-	from frappe.model import default_fields, optional_fields
+	from frappe.model import default_fields, optional_fields, child_table_fields
 
 	if isinstance(f, dict):
 		key, value = next(iter(f.items()))
@@ -1400,7 +1400,7 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) -
 		frappe.throw(frappe._("Operator must be one of {0}").format(", ".join(valid_operators)))
 
 
-	if f.doctype and (f.fieldname not in default_fields + optional_fields):
+	if f.doctype and (f.fieldname not in default_fields + optional_fields + child_table_fields):
 		# verify fieldname belongs to the doctype
 		meta = frappe.get_meta(f.doctype)
 		if not meta.has_field(f.fieldname):

frappe/website/doctype/web_form/web_form.py

@@ -77,7 +77,7 @@ class WebForm(WebsiteGenerator):
 
 			for prop in docfield_properties:
 				if df.fieldtype==meta_df.fieldtype and prop not in ("idx",
-					"reqd", "default", "description", "default", "options",
+					"reqd", "default", "description", "options",
 					"hidden", "read_only", "label"):
 					df.set(prop, meta_df.get(prop))
 

requirements.txt

@@ -21,7 +21,7 @@ googlemaps~=4.4.5
 gunicorn~=20.1.0
 html2text==2020.1.16
 html5lib~=1.1
-ipython~=7.27.0
+ipython~=7.31.1
 Jinja2~=3.0.1
 ldap3~=2.9
 markdown2~=2.4.0
@@ -32,7 +32,7 @@ openpyxl~=3.0.7
 passlib~=1.7.4
 paytmchecksum~=1.7.0
 pdfkit~=0.6.1
-Pillow~=8.2.0
+Pillow~=9.0.0
 premailer~=3.8.0
 psutil~=5.8.0
 psycopg2-binary~=2.9.1
@@ -63,7 +63,7 @@ sqlparse~=0.4.1
 stripe~=2.56.0
 terminaltables~=3.1.0
 urllib3~=1.26.4
-Werkzeug~=0.16.1
+Werkzeug~=2.0.3
 Whoosh~=2.7.4
 wrapt~=1.12.1
 xlrd~=2.0.1