Merge branch 'rebrand-ui' of https://github.com/frappe/frappe into rebrand-ui

.github/frappe_linter/translation.py

@@ -1,34 +0,0 @@
-import re
-import sys
-
-errors_encounter = 0
-pattern = re.compile(r"_\(([\"']{,3})(?P<message>((?!\1).)*)\1(\s*,\s*context\s*=\s*([\"'])(?P<py_context>((?!\5).)*)\5)*(\s*,\s*(.)*?\s*(,\s*([\"'])(?P<js_context>((?!\11).)*)\11)*)*\)")
-start_pattern = re.compile(r"_{1,2}\([\"']{1,3}")
-
-# skip first argument
-files = sys.argv[1:]
-files_to_scan = [_file for _file in files if _file.endswith(('.py', '.js'))]
-
-for _file in files_to_scan:
-	with open(_file, 'r') as f:
-		print(f'Checking: {_file}')
-		file_lines = f.readlines()
-		for line_number, line in enumerate(file_lines, 1):
-			start_matches = start_pattern.search(line)
-			if start_matches:
-				match = pattern.search(line)
-				if not match and line.endswith(',\n'):
-					# concat remaining text to validate multiline pattern
-					line = "".join(file_lines[line_number - 1:])
-					line = line[start_matches.start() + 1:]
-					match = pattern.match(line)
-
-				if not match:
-					errors_encounter += 1
-					print(f'\nTranslation syntax error at line number: {line_number + 1}\n{line.strip()[:100]}')
-
-if errors_encounter > 0:
-	print('\nYou can visit "https://frappeframework.com/docs/user/en/translations" to resolve this error.')
-	sys.exit(1)
-else:
-	print('\nGood To Go!')

.github/helper/documentation.py

@@ -0,0 +1,48 @@
+import sys
+import requests
+from urllib.parse import urlparse
+
+
+docs_repos = [
+	"frappe_docs",
+	"erpnext_documentation",
+	"erpnext_com",
+	"frappe_io",
+]
+
+
+def uri_validator(x):
+	result = urlparse(x)
+	return all([result.scheme, result.netloc, result.path])
+
+def docs_link_exists(body):
+	for line in body.splitlines():
+		for word in line.split():
+			if word.startswith('http') and uri_validator(word):
+				parsed_url = urlparse(word)
+				if parsed_url.netloc == "github.com":
+					_, org, repo, _type, ref = parsed_url.path.split('/')
+					if org == "frappe" and repo in docs_repos:
+						return True
+
+
+if __name__ == "__main__":
+	pr = sys.argv[1]
+	response = requests.get("https://api.github.com/repos/frappe/frappe/pulls/{}".format(pr))
+
+	if response.ok:
+		payload = response.json()
+		title = payload.get("title", "").lower()
+		head_sha = payload.get("head", {}).get("sha")
+		body = payload.get("body", "").lower()
+
+		if title.startswith("feat") and head_sha and "no-docs" not in body:
+			if docs_link_exists(body):
+				print("Documentation Link Found. You're Awesome! 🎉")
+
+			else:
+				print("Documentation Link Not Found! ⚠️")
+				sys.exit(1)
+
+		else:
+			print("Skipping documentation checks... 🏃")

.github/helper/translation.py

@@ -0,0 +1,60 @@
+import re
+import sys
+
+errors_encounter = 0
+pattern = re.compile(r"_\(([\"']{,3})(?P<message>((?!\1).)*)\1(\s*,\s*context\s*=\s*([\"'])(?P<py_context>((?!\5).)*)\5)*(\s*,\s*(.)*?\s*(,\s*([\"'])(?P<js_context>((?!\11).)*)\11)*)*\)")
+words_pattern = re.compile(r"_{1,2}\([\"'`]{1,3}.*?[a-zA-Z]")
+start_pattern = re.compile(r"_{1,2}\([f\"'`]{1,3}")
+f_string_pattern = re.compile(r"_\(f[\"']")
+starts_with_f_pattern = re.compile(r"_\(f")
+
+# skip first argument
+files = sys.argv[1:]
+files_to_scan = [_file for _file in files if _file.endswith(('.py', '.js'))]
+
+for _file in files_to_scan:
+	with open(_file, 'r') as f:
+		print(f'Checking: {_file}')
+		file_lines = f.readlines()
+		for line_number, line in enumerate(file_lines, 1):
+			if 'frappe-lint: disable-translate' in line:
+				continue
+
+			start_matches = start_pattern.search(line)
+			if start_matches:
+				starts_with_f = starts_with_f_pattern.search(line)
+
+				if starts_with_f:
+					has_f_string = f_string_pattern.search(line)
+					if has_f_string:
+						errors_encounter += 1
+						print(f'\nF-strings are not supported for translations at line number {line_number + 1}\n{line.strip()[:100]}')
+						continue
+					else:
+						continue
+
+				match = pattern.search(line)
+				error_found = False
+
+				if not match and line.endswith(',\n'):
+					# concat remaining text to validate multiline pattern
+					line = "".join(file_lines[line_number - 1:])
+					line = line[start_matches.start() + 1:]
+					match = pattern.match(line)
+
+				if not match:
+					error_found = True
+					print(f'\nTranslation syntax error at line number {line_number + 1}\n{line.strip()[:100]}')
+
+				if not error_found and not words_pattern.search(line):
+					error_found = True
+					print(f'\nTranslation is useless because it has no words at line number {line_number + 1}\n{line.strip()[:100]}')
+
+				if error_found:
+					errors_encounter += 1
+
+if errors_encounter > 0:
+	print('\nVisit "https://frappeframework.com/docs/user/en/translations" to learn about valid translation strings.')
+	sys.exit(1)
+else:
+	print('\nGood To Go!')

.github/workflows/docs-checker.yml

@@ -0,0 +1,24 @@
+name: 'Documentation Required'
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened, edited ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 'Setup Environment'
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: 'Clone repo'
+        uses: actions/checkout@v2
+
+      - name: Validate Docs
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          pip install requests --quiet
+          python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER

.github/workflows/translation_linter.yml

@@ -19,4 +19,4 @@ jobs:
       run: |
         git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
         files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        python $GITHUB_WORKSPACE/.github/frappe_linter/translation.py $files
+        python $GITHUB_WORKSPACE/.github/helper/translation.py $files

README.md

@@ -43,7 +43,7 @@ Full-stack web application framework that uses Python and MariaDB on the server
 
 ## Contributing
 
-1. [Pull Request Requirements](https://github.com/frappe/erpnext/wiki/Pull-Request-Guidelines)
+1. [Contribution Guidelines](https://github.com/frappe/erpnext/wiki/Contribution-Guidelines)
 1. [Translations](https://translate.erpnext.com)
 
 ### Website

cypress/integration/file_uploader.js

@@ -20,10 +20,14 @@ context('FileUploader', () => {
 		open_upload_dialog();
 
 		cy.fixture('example.json').then(fileContent => {
-			cy.get_open_dialog().find('.file-upload-area').upload(
-				{ fileContent, fileName: 'example.json', mimeType: 'application/json' },
-				{ subjectType: 'drag-n-drop' },
-			);
+			cy.get_open_dialog().find('.file-upload-area').upload({
+				fileContent,
+				fileName: 'example.json',
+				mimeType: 'application/json'
+			}, {
+				subjectType: 'drag-n-drop',
+				force: true
+			});
 			cy.get_open_dialog().find('.file-info').should('contain', 'example.json');
 			cy.server();
 			cy.route('POST', '/api/method/upload_file').as('upload_file');

frappe/automation/doctype/assignment_rule/assignment_rule.js

@@ -2,6 +2,9 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on('Assignment Rule', {
+	onload: (frm) => {
+		frm.trigger('set_due_date_field_options');
+	},
 	refresh: function(frm) {
 		// refresh description
 		frm.events.rule(frm);
@@ -12,5 +15,25 @@ frappe.ui.form.on('Assignment Rule', {
 		} else {
 			frm.get_field('rule').set_description(__('Assign to the one who has the least assignments'));
 		}
+	},
+	document_type: (frm) => {
+		frm.trigger('set_due_date_field_options');
+	},
+	set_due_date_field_options: (frm) => {
+		let doctype = frm.doc.document_type;
+		let datetime_fields = [];
+		if (doctype) {
+			frappe.model.with_doctype(doctype, () => {
+				frappe.get_meta(doctype).fields.map((df) => {
+					if (['Date', 'Datetime'].includes(df.fieldtype)) {
+						datetime_fields.push({ label: df.label, value: df.fieldname });
+					}
+				});
+				if (datetime_fields) {
+					frm.set_df_property('due_date_based_on', 'options', datetime_fields);
+				}
+				frm.set_df_property('due_date_based_on', 'hidden', !datetime_fields.length);
+			});
+		}
 	}
 });

frappe/automation/doctype/assignment_rule/assignment_rule.json

@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "allow_rename": 1,
  "autoname": "Prompt",
  "creation": "2019-02-28 17:12:18.815830",
@@ -8,6 +9,7 @@
  "engine": "InnoDB",
  "field_order": [
   "document_type",
+  "due_date_based_on",
   "priority",
   "disabled",
   "column_break_4",
@@ -129,9 +131,18 @@
    "label": "Assignment Days",
    "options": "Assignment Rule Day",
    "reqd": 1
+  },
+  {
+   "depends_on": "document_type",
+   "fieldname": "due_date_based_on",
+   "fieldtype": "Select",
+   "label": "Due Date Based On",
+   "description": "Value from this field will be set as the due date in the ToDo"
   }
  ],
- "modified": "2019-09-25 14:52:12.214514",
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2020-10-13 06:48:54.019735",
  "modified_by": "Administrator",
  "module": "Automation",
  "name": "Assignment Rule",

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -19,14 +19,14 @@ class AssignmentRule(Document):
 			repeated_days = get_repeated(assignment_days)
 			frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days)))
 
-	def on_update(self): # pylint: disable=no-self-use
-		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
+	def on_update(self):
+		clear_assignment_rule_cache(self)
 
-	def after_rename(self, old, new, merge): # pylint: disable=no-self-use
-		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
+	def after_rename(self, old, new, merge):
+		clear_assignment_rule_cache(self)
 
-	def on_trash(self): # pylint: disable=no-self-use
-		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
+	def on_trash(self):
+		clear_assignment_rule_cache(self)
 
 	def apply_unassign(self, doc, assignments):
 		if (self.unassign_condition and
@@ -53,7 +53,8 @@ class AssignmentRule(Document):
 			name = doc.get('name'),
 			description = frappe.render_template(self.description, doc),
 			assignment_rule = self.name,
-			notify = True
+			notify = True,
+			date = doc.get(self.due_date_based_on) if self.due_date_based_on else None
 		))
 
 		# set for reference in round robin
@@ -188,7 +189,7 @@ def apply(doc, method=None, doctype=None, name=None):
 
 	# multiple auto assigns
 	for d in assignment_rules:
-		assignment_rule_docs.append(frappe.get_doc('Assignment Rule', d.get('name')))
+		assignment_rule_docs.append(frappe.get_cached_doc('Assignment Rule', d.get('name')))
 
 	if not assignment_rule_docs:
 		return
@@ -237,6 +238,40 @@ def apply(doc, method=None, doctype=None, name=None):
 						break
 			assignment_rule.close_assignments(doc)
 
+def update_due_date(doc, state=None):
+	# called from hook
+	if (frappe.flags.in_patch
+		or frappe.flags.in_install
+		or frappe.flags.in_migrate
+		or frappe.flags.in_import
+		or frappe.flags.in_setup_wizard):
+		return
+	assignment_rules = frappe.cache_manager.get_doctype_map('Assignment Rule', 'due_date_rules_for_' + doc.doctype, dict(
+		document_type = doc.doctype,
+		disabled = 0,
+		due_date_based_on = ['is', 'set']
+	))
+	for rule in assignment_rules:
+		rule_doc = frappe.get_cached_doc('Assignment Rule', rule.get('name'))
+		due_date_field = rule_doc.due_date_based_on
+		if doc.meta.has_field(due_date_field) and \
+			doc.has_value_changed(due_date_field) and rule.get('name'):
+			assignment_todos = frappe.get_all('ToDo', {
+				'assignment_rule': rule.get('name'),
+				'status': 'Open',
+				'reference_type': doc.doctype,
+				'reference_name': doc.name
+			})
+			for todo in assignment_todos:
+				todo_doc = frappe.get_doc('ToDo', todo.name)
+				todo_doc.date = doc.get(due_date_field)
+				todo_doc.flags.updater_reference = {
+					'doctype': 'Assignment Rule',
+					'docname': rule.get('name'),
+					'label': _('via Assignment Rule')
+				}
+				todo_doc.save(ignore_permissions=True)
+
 def get_assignment_rules():
 	return [d.document_type for d in frappe.db.get_all('Assignment Rule', fields=['document_type'], filters=dict(disabled = 0))]
 
@@ -250,3 +285,7 @@ def get_repeated(values):
 			if value not in diff:
 				diff.append(str(value))
 	return " ".join(diff)
+
+def clear_assignment_rule_cache(rule):
+	frappe.cache_manager.clear_doctype_map('Assignment Rule', rule.document_type)
+	frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type)

frappe/automation/doctype/assignment_rule/test_assignment_rule.py

@@ -20,6 +20,7 @@ class TestAutoAssign(unittest.TestCase):
 			dict(day = 'Friday'),
 			dict(day = 'Saturday'),
 		]
+		self.days = days
 		self.assignment_rule = get_assignment_rule([days, days])
 		clear_assignments()
 
@@ -180,6 +181,55 @@ class TestAutoAssign(unittest.TestCase):
 			status = 'Open'
 		), 'owner'), ['test3@example.com'])
 
+	def test_assignment_rule_condition(self):
+		frappe.db.sql("DELETE FROM `tabAssignment Rule`")
+
+		# Add expiry_date custom field
+		from frappe.custom.doctype.custom_field.custom_field import create_custom_field
+		df = dict(fieldname='expiry_date', label='Expiry Date', fieldtype='Date')
+		create_custom_field('Note', df)
+
+		assignment_rule = frappe.get_doc(dict(
+			name = 'Assignment with Due Date',
+			doctype = 'Assignment Rule',
+			document_type = 'Note',
+			assign_condition = 'public == 0',
+			due_date_based_on = 'expiry_date',
+			assignment_days = self.days,
+			users = [
+				dict(user = 'test@example.com'),
+			]
+		)).insert()
+
+		expiry_date = frappe.utils.add_days(frappe.utils.nowdate(), 2)
+		note1 = make_note({'expiry_date': expiry_date})
+		note2 = make_note({'expiry_date': expiry_date})
+
+		note1_todo = frappe.get_all('ToDo', filters=dict(
+			reference_type = 'Note',
+			reference_name = note1.name,
+			status = 'Open'
+		))[0]
+
+		note1_todo_doc = frappe.get_doc('ToDo', note1_todo.name)
+		self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), expiry_date)
+
+		# due date should be updated if the reference doc's date is updated.
+		note1.expiry_date = frappe.utils.add_days(expiry_date, 2)
+		note1.save()
+		note1_todo_doc.reload()
+		self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), note1.expiry_date)
+
+		# saving one note's expiry should not update other note todo's due date
+		note2_todo = frappe.get_all('ToDo', filters=dict(
+			reference_type = 'Note',
+			reference_name = note2.name,
+			status = 'Open'
+		), fields=['name', 'date'])[0]
+		self.assertNotEqual(frappe.utils.get_date_str(note2_todo.date), note1.expiry_date)
+		self.assertEqual(frappe.utils.get_date_str(note2_todo.date), expiry_date)
+		assignment_rule.delete()
+
 def clear_assignments():
 	frappe.db.sql("delete from tabToDo where reference_type = 'Note'")
 

frappe/build.py

@@ -40,6 +40,7 @@ def build_missing_files():
 	# check which files dont exist yet from the build.json and tell build.js to build only those!
 	missing_assets = []
 	current_asset_files = []
+	frappe_build = os.path.join("..", "apps", "frappe", "frappe", "public", "build.json")
 
 	for type in ["css", "js"]:
 		current_asset_files.extend(
@@ -49,7 +50,7 @@ def build_missing_files():
 			]
 		)
 
-	with open(os.path.join(sites_path, "assets", "frappe", "build.json")) as f:
+	with open(frappe_build) as f:
 		all_asset_files = json.load(f).keys()
 
 	for asset in all_asset_files:
@@ -111,13 +112,21 @@ def download_frappe_assets(verbose=True):
 
 			if assets_archive:
 				import tarfile
+				directories_created = set()
 
 				click.secho("\nExtracting assets...\n", fg="yellow")
 				with tarfile.open(assets_archive) as tar:
 					for file in tar:
 						if not file.isdir():
 							dest = "." + file.name.replace("./frappe-bench/sites", "")
+							asset_directory = os.path.dirname(dest)
 							show = dest.replace("./assets/", "")
+
+							if asset_directory not in directories_created:
+								if not os.path.exists(asset_directory):
+									os.makedirs(asset_directory, exist_ok=True)
+								directories_created.add(asset_directory)
+
 							tar.makefile(file, dest)
 							print("{0} Restored {1}".format(green('✔'), show))
 

frappe/commands/site.py

@@ -1,10 +1,5 @@
 # imports - standard imports
-import atexit
-import compileall
-import hashlib
 import os
-import re
-import shutil
 import sys
 
 # imports - third party imports
@@ -13,9 +8,7 @@ import click
 # imports - module imports
 import frappe
 from frappe.commands import get_site, pass_context
-from frappe.commands.scheduler import _is_scheduler_enabled
 from frappe.exceptions import SiteNotSpecifiedError
-from frappe.installer import update_site_config
 from frappe.utils import get_site_path, touch_file
 
 
@@ -64,8 +57,10 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
 		sys.exit(1)
 
 	if not db_name:
+		import hashlib
 		db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16]
 
+	from frappe.commands.scheduler import _is_scheduler_enabled
 	from frappe.installer import install_db, make_site_dirs
 	from frappe.installer import install_app as _install_app
 	import frappe.utils.scheduler
@@ -73,6 +68,7 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
 	frappe.init(site=site)
 
 	try:
+
 		# enable scheduler post install?
 		enable_scheduler = _is_scheduler_enabled()
 	except Exception:
@@ -107,11 +103,11 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
 @click.option('--install-app', multiple=True, help='Install app after installation')
 @click.option('--with-public-files', help='Restores the public files of the site, given path to its tar file')
 @click.option('--with-private-files', help='Restores the private files of the site, given path to its tar file')
-@click.option('--force', is_flag=True, default=False, help='Use a bit of force to get the job done')
+@click.option('--force', is_flag=True, default=False, help='Ignore the site downgrade warning, if applicable')
 @pass_context
 def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None):
 	"Restore site database from an sql file"
-	from frappe.installer import extract_sql_gzip, extract_tar_files, is_downgrade
+	from frappe.installer import extract_sql_gzip, extract_files, is_downgrade
 	force = context.force or force
 
 	# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
@@ -147,12 +143,12 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
 	# Extract public and/or private files to the restored site, if user has given the path
 	if with_public_files:
 		with_public_files = os.path.join(base_path, with_public_files)
-		public = extract_tar_files(site, with_public_files, 'public')
+		public = extract_files(site, with_public_files, 'public')
 		os.remove(public)
 
 	if with_private_files:
 		with_private_files = os.path.join(base_path, with_private_files)
-		private = extract_tar_files(site, with_private_files, 'private')
+		private = extract_files(site, with_private_files, 'private')
 		os.remove(private)
 
 	# Removing temporarily created file
@@ -276,6 +272,8 @@ def disable_user(context, email):
 @pass_context
 def migrate(context, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
+	import compileall
+	import re
 	from frappe.migrate import migrate
 
 	for site in context.sites:
@@ -385,35 +383,34 @@ def use(site, sites_path='.'):
 
 @click.command('backup')
 @click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
-@click.option('--verbose', default=False, is_flag=True)
+@click.option('--backup-path', default=None, help="Set path for saving all the files in this operation")
+@click.option('--backup-path-db', default=None, help="Set path for saving database file")
+@click.option('--backup-path-files', default=None, help="Set path for saving public file")
+@click.option('--backup-path-private-files', default=None, help="Set path for saving private file")
+@click.option('--backup-path-conf', default=None, help="Set path for saving config file")
+@click.option('--verbose', default=False, is_flag=True, help="Add verbosity")
+@click.option('--compress', default=False, is_flag=True, help="Compress private and public files")
 @pass_context
-def backup(context, with_files=False, backup_path_db=None, backup_path_files=None,
-	backup_path_private_files=None, quiet=False, verbose=False):
+def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None,
+	backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False):
 	"Backup"
 	from frappe.utils.backups import scheduled_backup
 	verbose = verbose or context.verbose
 	exit_code = 0
+
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, force=True, verbose=verbose)
-		except Exception as e:
-			if verbose:
-				print("Backup failed for {0}. Database or site_config.json may be corrupted".format(site))
+			odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress)
+		except Exception:
+			click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red")
 			exit_code = 1
 			continue
-
-		if verbose:
-			from frappe.utils import now
-			summary_title = "Backup Summary at {0}".format(now())
-			print(summary_title + "\n" + "-" * len(summary_title))
-			print("Database backup:", odb.backup_path_db)
-			if with_files:
-				print("Public files:   ", odb.backup_path_files)
-				print("Private files:  ", odb.backup_path_private_files)
-
+		odb.print_summary()
+		click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green")
 		frappe.destroy()
+
 	if not context.sites:
 		raise SiteNotSpecifiedError
 

frappe/commands/utils.py

@@ -554,10 +554,24 @@ def run_ui_tests(context, app, headless=False):
 	site_env = 'CYPRESS_baseUrl={}'.format(site_url)
 	password_env = 'CYPRESS_adminPassword={}'.format(admin_password) if admin_password else ''
 
+	os.chdir(app_base_path)
+
+	node_bin = subprocess.getoutput("npm bin")
+	cypress_path = "{0}/cypress".format(node_bin)
+	plugin_path = "{0}/cypress-file-upload".format(node_bin)
+
+	# check if cypress in path...if not, install it.
+	if not (os.path.exists(cypress_path) or os.path.exists(plugin_path)):
+		# install cypress
+		click.secho("Installing Cypress...", fg="yellow")
+		frappe.commands.popen("yarn add cypress@3 cypress-file-upload@^3.1 --no-lockfile")
+
 	# run for headless mode
 	run_or_open = 'run --browser chrome --record --key 4a48f41c-11b3-425b-aa88-c58048fa69eb' if headless else 'open'
-	command = '{site_env} {password_env} yarn run cypress {run_or_open}'
-	formatted_command = command.format(site_env=site_env, password_env=password_env, run_or_open=run_or_open)
+	command = '{site_env} {password_env} {cypress} {run_or_open}'
+	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
+
+	click.secho("Running Cypress...", fg="yellow")
 	frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
 
 

frappe/config/settings.py

@@ -23,6 +23,11 @@ def get_data():
 					"description": _("Company, Fiscal Year and Currency defaults"),
 					"hide_count": True
 				},
+				{
+					"type": "doctype",
+					"name": "Log Settings",
+					"description": _("Log cleanup and notification configuration")
+				},
 				{
 					"type": "doctype",
 					"name": "Error Log",

frappe/contacts/doctype/address/address.json

@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "allow_import": 1,
  "allow_rename": 1,
  "creation": "2013-01-10 16:34:32",
@@ -24,7 +25,6 @@
   "is_shipping_address",
   "disabled",
   "linked_with",
-  "is_your_company_address",
   "links"
  ],
  "fields": [
@@ -138,12 +138,6 @@
    "label": "Reference",
    "options": "fa fa-pushpin"
   },
-  {
-   "default": "0",
-   "fieldname": "is_your_company_address",
-   "fieldtype": "Check",
-   "label": "Is Your Company Address"
-  },
   {
    "fieldname": "links",
    "fieldtype": "Table",
@@ -153,7 +147,8 @@
  ],
  "icon": "fa fa-map-marker",
  "idx": 5,
- "modified": "2019-09-08 11:41:04.145589",
+ "links": [],
+ "modified": "2020-10-14 17:38:08.971776",
  "modified_by": "Administrator",
  "module": "Contacts",
  "name": "Address",

frappe/contacts/doctype/address/address.py

@@ -39,14 +39,13 @@ class Address(Document):
 
 	def validate(self):
 		self.link_address()
-		self.validate_reference()
 		self.validate_preferred_address()
 		set_link_title(self)
 		deduplicate_dynamic_links(self)
 
 	def link_address(self):
 		"""Link address based on owner"""
-		if not self.links and not self.is_your_company_address:
+		if not self.links:
 			contact_name = frappe.db.get_value("Contact", {"email_id": self.owner})
 			if contact_name:
 				contact = frappe.get_cached_doc('Contact', contact_name)
@@ -56,12 +55,6 @@ class Address(Document):
 
 		return False
 
-	def validate_reference(self):
-		if self.is_your_company_address:
-			if not [row for row in self.links if row.link_doctype == "Company"]:
-				frappe.throw(_("Address needs to be linked to a Company. Please add a row for Company in the Links table below."),
-					title =_("Company not Linked"))
-
 	def validate_preferred_address(self):
 		preferred_fields = ['is_primary_address', 'is_shipping_address']
 
@@ -204,25 +197,6 @@ def get_address_templates(address):
 	else:
 		return result
 
-@frappe.whitelist()
-def get_shipping_address(company, address = None):
-	filters = [
-		["Dynamic Link", "link_doctype", "=", "Company"],
-		["Dynamic Link", "link_name", "=", company],
-		["Address", "is_your_company_address", "=", 1]
-	]
-	fields = ["*"]
-	if address and frappe.db.get_value('Dynamic Link',
-		{'parent': address, 'link_name': company}):
-		filters.append(["Address", "name", "=", address])
-
-	address = frappe.get_all("Address", filters=filters, fields=fields) or {}
-
-	if address:
-		address_as_dict = address[0]
-		name, address_template = get_address_templates(address_as_dict)
-		return address_as_dict.get("name"), frappe.render_template(address_template, address_as_dict)
-
 def get_company_address(company):
 	ret = frappe._dict()
 	ret.company_address = get_default_address('Company', company)

frappe/core/doctype/activity_log/activity_log.py

@@ -40,7 +40,11 @@ def add_authentication_log(subject, user, operation="Login", status="Success"):
 		"operation": operation,
 	}).insert(ignore_permissions=True, ignore_links=True)
 
-def clear_authentication_logs():
-	"""clear 100 day old authentication logs"""
+def clear_activity_logs(days=None):
+	"""clear 90 day old authentication logs or configured in log settings"""
+
+	if not days:
+		days = 90
+
 	frappe.db.sql("""delete from `tabActivity Log` where \
-	creation< (NOW() - INTERVAL '100' DAY)""")
+		creation< (NOW() - INTERVAL '{0}' DAY)""".format(days))

frappe/core/doctype/communication/email.py

@@ -55,7 +55,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 	comm = frappe.get_doc({
 		"doctype":"Communication",
 		"subject": subject,
-		"content": content,
+		"content": frappe.utils.sanitize_html(content),
 		"sender": sender,
 		"sender_full_name":sender_full_name,
 		"recipients": recipients,

frappe/core/doctype/error_log/error_log.py

@@ -17,9 +17,6 @@ def set_old_logs_as_seen():
 	frappe.db.sql("""UPDATE `tabError Log` SET `seen`=1
 		WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""")
 
-	# clear old logs
-	frappe.db.sql("""DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '30' DAY)""")
-
 @frappe.whitelist()
 def clear_error_logs():
 	'''Flush all Error Logs'''

frappe/core/doctype/log_setting_user/log_setting_user.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Log Setting User', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/log_setting_user/log_setting_user.json

@@ -0,0 +1,34 @@
+{
+ "actions": [],
+ "autoname": "field:user",
+ "creation": "2020-10-08 13:09:36.034430",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "user"
+ ],
+ "fields": [
+  {
+   "fieldname": "user",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "User",
+   "options": "User",
+   "reqd": 1,
+   "unique": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2020-10-08 17:22:04.690348",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Log Setting User",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/log_setting_user/log_setting_user.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class LogSettingUser(Document):
+	pass

frappe/core/doctype/log_setting_user/test_log_setting_user.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestLogSettingUser(unittest.TestCase):
+	pass

frappe/core/doctype/log_settings/log_settings.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Log Settings', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/log_settings/log_settings.json

@@ -0,0 +1,83 @@
+{
+ "actions": [],
+ "creation": "2020-10-08 12:12:21.694424",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "error_log_notification_section",
+  "users_to_notify",
+  "log_cleanup_section",
+  "clear_error_log_after",
+  "clear_activity_log_after",
+  "column_break_4",
+  "clear_email_queue_after"
+ ],
+ "fields": [
+  {
+   "fieldname": "log_cleanup_section",
+   "fieldtype": "Section Break",
+   "label": "Log Cleanup"
+  },
+  {
+   "fieldname": "column_break_4",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "error_log_notification_section",
+   "fieldtype": "Section Break",
+   "label": "Error Log Notification"
+  },
+  {
+   "fieldname": "users_to_notify",
+   "fieldtype": "Table MultiSelect",
+   "label": "Users To Notify",
+   "options": "Log Setting User"
+  },
+  {
+   "default": "90",
+   "description": "In Days",
+   "fieldname": "clear_error_log_after",
+   "fieldtype": "Int",
+   "label": "Clear Error log After"
+  },
+  {
+   "default": "90",
+   "description": "In Days",
+   "fieldname": "clear_activity_log_after",
+   "fieldtype": "Int",
+   "label": "Clear Activity Log After"
+  },
+  {
+   "default": "90",
+   "description": "In Days",
+   "fieldname": "clear_email_queue_after",
+   "fieldtype": "Int",
+   "label": "Clear Email Queue After"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-10-13 12:18:48.649038",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Log Settings",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/log_settings/log_settings.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe import _
+from frappe.model.document import Document
+
+class LogSettings(Document):
+	def clear_logs(self):
+		self.clear_error_logs()
+		self.clear_activity_logs()
+		self.clear_email_queue()
+
+	def clear_error_logs(self):
+		frappe.db.sql(""" DELETE FROM `tabError Log`
+			WHERE `creation` < (NOW() - INTERVAL '{0}' DAY)
+		""".format(self.clear_error_log_after))
+
+	def clear_activity_logs(self):
+		from frappe.core.doctype.activity_log.activity_log import clear_activity_logs
+		clear_activity_logs(days=self.clear_activity_log_after)
+
+	def clear_email_queue(self):
+		from frappe.email.queue import clear_outbox
+		clear_outbox(days=self.clear_email_queue_after)
+
+def run_log_clean_up():
+	doc = frappe.get_doc("Log Settings")
+	doc.clear_logs()
+
+@frappe.whitelist()
+def has_unseen_error_log(user):
+
+	def _get_response(show_alert=True):
+		return {
+			'show_alert': True,
+			'message': _("You have unseen {0}").format('<a href="/desk#List/Error%20Log/List"> Error Logs </a>')
+		}
+
+	if frappe.db.sql_list("select name from `tabError Log` where seen = 0 limit 1"):
+		log_settings = frappe.get_cached_doc('Log Settings')
+
+		if log_settings.users_to_notify:
+			if user in [u.user for u in log_settings.users_to_notify]:
+				return _get_response()
+			else:
+				return _get_response(show_alert=False)
+		else:
+			return _get_response()

frappe/core/doctype/log_settings/test_log_settings.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestLogSettings(unittest.TestCase):
+	pass

frappe/core/doctype/report/report.py

@@ -49,6 +49,8 @@ class Report(Document):
 		self.export_doc()
 
 	def on_trash(self):
+		if self.is_standard == 'Yes' and not cint(getattr(frappe.local.conf, 'developer_mode',0)):
+			frappe.throw(_("You are not allowed to delete Standard Report"))
 		delete_custom_role('report', self.name)
 
 	def get_columns(self):

frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py

@@ -22,7 +22,7 @@ class TestScheduledJobType(unittest.TestCase):
 		self.assertEqual(all_job.frequency, 'All')
 
 		daily_job = frappe.get_doc('Scheduled Job Type',
-			dict(method='frappe.email.queue.clear_outbox'))
+			dict(method='frappe.email.queue.set_expiry_for_email_queue'))
 		self.assertEqual(daily_job.frequency, 'Daily')
 
 		# check if cron jobs are synced
@@ -38,7 +38,7 @@ class TestScheduledJobType(unittest.TestCase):
 		self.assertEqual(updated_scheduled_job.frequency, "Hourly")
 
 	def test_daily_job(self):
-		job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.clear_outbox'))
+		job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.set_expiry_for_email_queue'))
 		job.db_set('last_execution', '2019-01-01 00:00:00')
 		self.assertTrue(job.is_event_due(get_datetime('2019-01-02 00:00:06')))
 		self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:00:06')))

frappe/custom/doctype/custom_field/custom_field.py

@@ -32,6 +32,7 @@ class CustomField(Document):
 		self.fieldname = self.fieldname.lower()
 
 	def before_insert(self):
+		self.set_fieldname()
 		meta = frappe.get_meta(self.dt, cached=False)
 		fieldnames = [df.fieldname for df in meta.get("fields")]
 

frappe/data/sample_site_config.json

@@ -22,6 +22,9 @@
  "use_ssl": 0,
  "auto_email_id": "hello@example.com",
 
+ "google_analytics_id": "google_analytics_id",
+ "google_analytics_anonymize_ip": 1,
+
  "google_login": {
 	 "client_id": "google_client_id",
 	 "client_secret": "google_client_secret"

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js

@@ -30,7 +30,7 @@ frappe.ui.form.on('Data Migration Connector', {
 							frm.set_value('connector_type', 'Custom');
 							frm.set_value('python_module', r.message);
 							frm.save();
-							frappe.show_alert({message: __(`New module created ${r.message}`), indicator: 'success'});
+							frappe.show_alert(__("New module created {0}", [r.message]));
 							d.hide();
 						}
 					});

frappe/database/postgres/setup_db.py

@@ -20,8 +20,11 @@ def setup_database(force, source_sql=None, verbose=False):
 		source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql')
 
 	subprocess.check_output([
-		'psql', frappe.conf.db_name, '-h', frappe.conf.db_host or 'localhost', '-U',
-		frappe.conf.db_name, '-f', source_sql
+		'psql', frappe.conf.db_name,
+		'-h', frappe.conf.db_host or 'localhost',
+		'-p', str(frappe.conf.db_port or '5432'),
+		'-U', frappe.conf.db_name,
+		'-f', source_sql
 	], env=subprocess_env)
 
 	frappe.connect()

frappe/desk/doctype/dashboard_chart/dashboard_chart.js

@@ -169,7 +169,7 @@ frappe.ui.form.on('Dashboard Chart', {
 					frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data);
 					frm.set_df_property('x_field', 'options', frm.field_options.non_numeric_fields);
 					if (!frm.field_options.numeric_fields.length) {
-						frappe.msgprint(__(`Report has no numeric fields, please change the Report Name`));
+						frappe.msgprint(__("Report has no numeric fields, please change the Report Name"));
 					} else {
 						let y_field_df = frappe.meta.get_docfield('Dashboard Chart Field', 'y_field', frm.doc.name);
 						y_field_df.options = frm.field_options.numeric_fields;

frappe/desk/doctype/module_onboarding/module_onboarding.py

@@ -41,3 +41,15 @@ class ModuleOnboarding(Document):
 
 	def before_export(self, doc):
 		doc.is_complete = 0
+
+	def reset_onboarding(self):
+		frappe.only_for("Administrator")
+
+		self.is_complete = 0
+		steps = self.get_steps()
+		for step in steps:
+			step.is_complete = 0
+			step.is_skipped = 0
+			step.save()
+
+		self.save()

frappe/desk/doctype/number_card/number_card.js

@@ -207,7 +207,7 @@ frappe.ui.form.on('Number Card', {
 				frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data);
 				frm.set_df_property('report_field', 'options', frm.field_options.numeric_fields);
 				if (!frm.field_options.numeric_fields.length) {
-					frappe.msgprint(__(`Report has no numeric fields, please change the Report Name`));
+					frappe.msgprint(__("Report has no numeric fields, please change the Report Name"));
 				}
 			} else {
 				frappe.msgprint(__('Report has no data, please modify the filters or change the Report Name'));

frappe/desk/page/user_profile/user_profile.js

@@ -42,7 +42,7 @@ class UserProfile {
 	}
 
 	make_user_profile() {
-		frappe.set_route('user-profile', this.user_id);
+		frappe.set_route('user-profile', this.user_id, { redirect: true });
 		this.user = frappe.user_info(this.user_id);
 		this.page.set_title(this.user.fullname);
 		this.setup_user_search();

frappe/desk/query_report.py

@@ -4,11 +4,19 @@
 from __future__ import unicode_literals
 
 import frappe
-import os, json
+import os
+import json
 
 from frappe import _
 from frappe.modules import scrub, get_module_path
-from frappe.utils import flt, cint, get_html_format, get_url_to_form, gzip_decompress, format_duration
+from frappe.utils import (
+	flt,
+	cint,
+	get_html_format,
+	get_url_to_form,
+	gzip_decompress,
+	format_duration,
+)
 from frappe.model.utils import render_include
 from frappe.translate import send_translations
 import frappe.desk.reportview
@@ -17,11 +25,12 @@ from six import string_types, iteritems
 from datetime import timedelta
 from frappe.core.utils import ljust_list
 
+
 def get_report_doc(report_name):
 	doc = frappe.get_doc("Report", report_name)
 	doc.custom_columns = []
 
-	if doc.report_type == 'Custom Report':
+	if doc.report_type == "Custom Report":
 		custom_report_doc = doc
 		reference_report = custom_report_doc.reference_report
 		doc = frappe.get_doc("Report", reference_report)
@@ -30,11 +39,18 @@ def get_report_doc(report_name):
 		doc.is_custom_report = True
 
 	if not doc.is_permitted():
-		frappe.throw(_("You don't have access to Report: {0}").format(report_name), frappe.PermissionError)
+		frappe.throw(
+			_("You don't have access to Report: {0}").format(report_name),
+			frappe.PermissionError,
+		)
 
 	if not frappe.has_permission(doc.ref_doctype, "report"):
-		frappe.throw(_("You don't have permission to get a report on: {0}").format(doc.ref_doctype),
-			frappe.PermissionError)
+		frappe.throw(
+			_("You don't have permission to get a report on: {0}").format(
+				doc.ref_doctype
+			),
+			frappe.PermissionError,
+		)
 
 	if doc.disabled:
 		frappe.throw(_("Report {0} is disabled").format(report_name))
@@ -54,11 +70,10 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None)
 	if report.report_type == "Query Report":
 		res = report.execute_query_report(filters)
 
-	elif report.report_type == 'Script Report':
+	elif report.report_type == "Script Report":
 		res = report.execute_script_report(filters)
 
-	columns, result, message, chart, report_summary, skip_total_row = \
-		ljust_list(res, 6)
+	columns, result, message, chart, report_summary, skip_total_row = ljust_list(res, 6)
 
 	if report.custom_columns:
 		# Original query columns, needed to reorder data as per custom columns
@@ -74,7 +89,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None)
 		result = add_data_to_custom_columns(custom_columns, result)
 
 		for custom_column in custom_columns:
-			columns.insert(custom_column['insert_after_index'] + 1, custom_column)
+			columns.insert(custom_column["insert_after_index"] + 1, custom_column)
 
 	if result:
 		result = get_filtered_data(report.ref_doctype, columns, result, user)
@@ -90,17 +105,19 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None)
 		"report_summary": report_summary,
 		"skip_total_row": skip_total_row or 0,
 		"status": None,
-		"execution_time": frappe.cache().hget('report_execution_time', report.name) or 0
+		"execution_time": frappe.cache().hget("report_execution_time", report.name)
+		or 0,
 	}
 
+
 @frappe.whitelist()
 def background_enqueue_run(report_name, filters=None, user=None):
 	"""run reports in background"""
 	if not user:
 		user = frappe.session.user
 	report = get_report_doc(report_name)
-	track_instance = \
-		frappe.get_doc({
+	track_instance = frappe.get_doc(
+		{
 			"doctype": "Prepared Report",
 			"report_name": report_name,
 			# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
@@ -110,21 +127,24 @@ def background_enqueue_run(report_name, filters=None, user=None):
 			"report_type": report.report_type,
 			"query": report.query,
 			"module": report.module,
-		})
+		}
+	)
 	track_instance.insert(ignore_permissions=True)
 	frappe.db.commit()
 	track_instance.enqueue_report()
 
 	return {
 		"name": track_instance.name,
-		"redirect_url": get_url_to_form("Prepared Report", track_instance.name)
+		"redirect_url": get_url_to_form("Prepared Report", track_instance.name),
 	}
 
 
 @frappe.whitelist()
 def get_script(report_name):
 	report = get_report_doc(report_name)
-	module = report.module or frappe.db.get_value("DocType", report.ref_doctype, "module")
+	module = report.module or frappe.db.get_value(
+		"DocType", report.ref_doctype, "module"
+	)
 	module_path = get_module_path(module)
 	report_folder = os.path.join(module_path, "report", scrub(report.name))
 	script_path = os.path.join(report_folder, scrub(report.name) + ".js")
@@ -150,24 +170,38 @@ def get_script(report_name):
 	return {
 		"script": render_include(script),
 		"html_format": html_format,
-		"execution_time": frappe.cache().hget('report_execution_time', report_name) or 0
+		"execution_time": frappe.cache().hget("report_execution_time", report_name)
+		or 0,
 	}
 
 
 @frappe.whitelist()
 @frappe.read_only()
-def run(report_name, filters=None, user=None, ignore_prepared_report=False, custom_columns=None):
+def run(
+	report_name,
+	filters=None,
+	user=None,
+	ignore_prepared_report=False,
+	custom_columns=None,
+):
 
 	report = get_report_doc(report_name)
 	if not user:
 		user = frappe.session.user
 	if not frappe.has_permission(report.ref_doctype, "report"):
-		frappe.msgprint(_("Must have report permission to access this report."),
-			raise_exception=True)
+		frappe.msgprint(
+			_("Must have report permission to access this report."),
+			raise_exception=True,
+		)
 
 	result = None
 
-	if report.prepared_report and not report.disable_prepared_report and not ignore_prepared_report and not custom_columns:
+	if (
+		report.prepared_report
+		and not report.disable_prepared_report
+		and not ignore_prepared_report
+		and not custom_columns
+	):
 		if filters:
 			if isinstance(filters, string_types):
 				filters = json.loads(filters)
@@ -180,10 +214,13 @@ def run(report_name, filters=None, user=None, ignore_prepared_report=False, cust
 	else:
 		result = generate_report_result(report, filters, user, custom_columns)
 
-	result["add_total_row"] = report.add_total_row and not result.get('skip_total_row', False)
+	result["add_total_row"] = report.add_total_row and not result.get(
+		"skip_total_row", False
+	)
 
 	return result
 
+
 def add_data_to_custom_columns(columns, result):
 	custom_fields_data = get_data_for_custom_report(columns)
 
@@ -195,25 +232,28 @@ def add_data_to_custom_columns(columns, result):
 
 		if isinstance(row, list):
 			for idx, column in enumerate(columns):
-				if column.get('link_field'):
-					row_obj[column['fieldname']] = None
+				if column.get("link_field"):
+					row_obj[column["fieldname"]] = None
 					row.insert(idx, None)
 				else:
-					row_obj[column['fieldname']] = row[idx]
+					row_obj[column["fieldname"]] = row[idx]
 			data.append(row_obj)
 		else:
 			data.append(row)
 
 	for row in data:
 		for column in columns:
-			if column.get('link_field'):
-				fieldname = column['fieldname']
-				key = (column['doctype'], fieldname)
-				link_field = column['link_field']
-				row[fieldname] = custom_fields_data.get(key, {}).get(row.get(link_field))
+			if column.get("link_field"):
+				fieldname = column["fieldname"]
+				key = (column["doctype"], fieldname)
+				link_field = column["link_field"]
+				row[fieldname] = custom_fields_data.get(key, {}).get(
+					row.get(link_field)
+				)
 
 	return data
 
+
 def reorder_data_for_custom_columns(custom_columns, columns, result):
 	if not result:
 		return []
@@ -228,6 +268,7 @@ def reorder_data_for_custom_columns(custom_columns, columns, result):
 		# columns do not need to be reordered if result is a list of dicts
 		return result
 
+
 def get_columns_from_list(columns, target_columns, result):
 	reordered_result = []
 
@@ -244,6 +285,7 @@ def get_columns_from_list(columns, target_columns, result):
 
 	return reordered_result
 
+
 def get_prepared_report_result(report, filters, dn="", user=None):
 	latest_report_data = {}
 	doc = None
@@ -252,14 +294,15 @@ def get_prepared_report_result(report, filters, dn="", user=None):
 		doc = frappe.get_doc("Prepared Report", dn)
 	else:
 		# Only look for completed prepared reports with given filters.
-		doc_list = frappe.get_all("Prepared Report",
+		doc_list = frappe.get_all(
+			"Prepared Report",
 			filters={
 				"status": "Completed",
 				"filters": json.dumps(filters),
 				"owner": user,
-				"report_name": report.get('custom_report') or report.get('report_name')
+				"report_name": report.get("custom_report") or report.get("report_name"),
 			},
-			order_by = 'creation desc'
+			order_by="creation desc",
 		)
 
 		if doc_list:
@@ -269,11 +312,15 @@ def get_prepared_report_result(report, filters, dn="", user=None):
 	if doc:
 		try:
 			# Prepared Report data is stored in a GZip compressed JSON file
-			attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name")
-			attached_file = frappe.get_doc('File', attached_file_name)
+			attached_file_name = frappe.db.get_value(
+				"File",
+				{"attached_to_doctype": doc.doctype, "attached_to_name": doc.name},
+				"name",
+			)
+			attached_file = frappe.get_doc("File", attached_file_name)
 			compressed_content = attached_file.get_content()
 			uncompressed_content = gzip_decompress(compressed_content)
-			data = json.loads(uncompressed_content)
+			data = json.loads(uncompressed_content.decode("utf-8"))
 			if data:
 				columns = json.loads(doc.columns) if doc.columns else data[0]
 
@@ -281,23 +328,18 @@ def get_prepared_report_result(report, filters, dn="", user=None):
 					if isinstance(column, dict) and column.get("label"):
 						column["label"] = _(column["label"])
 
-				latest_report_data = {
-					"columns": columns,
-					"result": data
-				}
+				latest_report_data = {"columns": columns, "result": data}
 		except Exception:
 			frappe.log_error(frappe.get_traceback())
 			frappe.delete_doc("Prepared Report", doc.name)
 			frappe.db.commit()
 			doc = None
 
-	latest_report_data.update({
-		"prepared_report": True,
-		"doc": doc
-	})
+	latest_report_data.update({"prepared_report": True, "doc": doc})
 
 	return latest_report_data
 
+
 @frappe.whitelist()
 def export_query():
 	"""export from query reports"""
@@ -313,8 +355,8 @@ def export_query():
 	if isinstance(data.get("report_name"), string_types):
 		report_name = data["report_name"]
 		frappe.permissions.can_export(
-			frappe.get_cached_value('Report', report_name, 'ref_doctype'),
-			raise_exception=True
+			frappe.get_cached_value("Report", report_name, "ref_doctype"),
+			raise_exception=True,
 		)
 	if isinstance(data.get("file_format_type"), string_types):
 		file_format_type = data["file_format_type"]
@@ -331,20 +373,26 @@ def export_query():
 		data = run(report_name, filters, custom_columns=custom_columns)
 		data = frappe._dict(data)
 		if not data.columns:
-			frappe.respond_as_web_page(_("No data to export"),
-			_("You can try changing the filters of your report."))
+			frappe.respond_as_web_page(
+				_("No data to export"),
+				_("You can try changing the filters of your report."),
+			)
 			return
 
 		columns = get_columns_dict(data.columns)
 
 		from frappe.utils.xlsxutils import make_xlsx
-		data['result'] = handle_duration_fieldtype_values(data.get('result'), data.get('columns'))
+
+		data["result"] = handle_duration_fieldtype_values(
+			data.get("result"), data.get("columns")
+		)
 		xlsx_data = build_xlsx_data(columns, data, visible_idx, include_indentation)
 		xlsx_file = make_xlsx(xlsx_data, "Query Report")
 
-		frappe.response['filename'] = report_name + '.xlsx'
-		frappe.response['filecontent'] = xlsx_file.getvalue()
-		frappe.response['type'] = 'binary'
+		frappe.response["filename"] = report_name + ".xlsx"
+		frappe.response["filecontent"] = xlsx_file.getvalue()
+		frappe.response["type"] = "binary"
+
 
 def handle_duration_fieldtype_values(result, columns):
 	for i, col in enumerate(columns):
@@ -370,6 +418,7 @@ def handle_duration_fieldtype_values(result, columns):
 
 	return result
 
+
 def build_xlsx_data(columns, data, visible_idx, include_indentation):
 	result = [[]]
 
@@ -386,13 +435,13 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation):
 
 			if isinstance(row, dict) and row:
 				for idx in range(len(data.columns)):
-					# check if column is not hidden 
+					# check if column is not hidden
 					if not columns[idx].get("hidden"):
 						label = columns[idx]["label"]
 						fieldname = columns[idx]["fieldname"]
 						cell_value = row.get(fieldname, row.get(label, ""))
-						if cint(include_indentation) and 'indent' in row and idx == 0:
-							cell_value = ('    ' * cint(row['indent'])) + cell_value
+						if cint(include_indentation) and "indent" in row and idx == 0:
+							cell_value = ("    " * cint(row["indent"])) + cell_value
 						row_data.append(cell_value)
 			else:
 				row_data = row
@@ -401,8 +450,9 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation):
 
 	return result
 
-def add_total_row(result, columns, meta = None):
-	total_row = [""]*len(columns)
+
+def add_total_row(result, columns, meta=None):
+	total_row = [""] * len(columns)
 	has_percent = []
 	for i, col in enumerate(columns):
 		fieldtype, options, fieldname = None, None, None
@@ -428,10 +478,13 @@ def add_total_row(result, columns, meta = None):
 			options = col.get("options")
 
 		for row in result:
-			if i >= len(row): continue
+			if i >= len(row):
+				continue
 
 			cell = row.get(fieldname) if isinstance(row, dict) else row[i]
-			if fieldtype in ["Currency", "Int", "Float", "Percent", "Duration"] and flt(cell):
+			if fieldtype in ["Currency", "Int", "Float", "Percent", "Duration"] and flt(
+				cell
+			):
 				total_row[i] = flt(total_row[i]) + flt(cell)
 
 			if fieldtype == "Percent" and i not in has_percent:
@@ -439,12 +492,15 @@ def add_total_row(result, columns, meta = None):
 
 			if fieldtype == "Time" and cell:
 				if not total_row[i]:
-					total_row[i]=timedelta(hours=0,minutes=0,seconds=0)
-				total_row[i] =  total_row[i] + cell
+					total_row[i] = timedelta(hours=0, minutes=0, seconds=0)
+				total_row[i] = total_row[i] + cell
 
-
-		if fieldtype=="Link" and options == "Currency":
-			total_row[i] = result[0].get(fieldname) if isinstance(result[0], dict) else result[0][i]
+		if fieldtype == "Link" and options == "Currency":
+			total_row[i] = (
+				result[0].get(fieldname)
+				if isinstance(result[0], dict)
+				else result[0][i]
+			)
 
 	for i in has_percent:
 		total_row[i] = flt(total_row[i]) / len(result)
@@ -463,35 +519,44 @@ def add_total_row(result, columns, meta = None):
 	result.append(total_row)
 	return result
 
+
 @frappe.whitelist()
 def get_data_for_custom_field(doctype, field):
 
 	if not frappe.has_permission(doctype, "read"):
 		frappe.throw(_("Not Permitted"), frappe.PermissionError)
 
-	value_map = frappe._dict(frappe.get_all(doctype,
-		fields=["name", field],
-		as_list=1))
+	value_map = frappe._dict(frappe.get_all(doctype, fields=["name", field], as_list=1))
 
 	return value_map
 
+
 def get_data_for_custom_report(columns):
 	doc_field_value_map = {}
 
 	for column in columns:
-		if column.get('link_field'):
-			fieldname = column.get('fieldname')
-			doctype = column.get('doctype')
-			doc_field_value_map[(doctype, fieldname)] = get_data_for_custom_field(doctype, fieldname)
+		if column.get("link_field"):
+			fieldname = column.get("fieldname")
+			doctype = column.get("doctype")
+			doc_field_value_map[(doctype, fieldname)] = get_data_for_custom_field(
+				doctype, fieldname
+			)
 
 	return doc_field_value_map
 
+
 @frappe.whitelist()
 def save_report(reference_report, report_name, columns):
 	report_doc = get_report_doc(reference_report)
 
-	docname = frappe.db.exists("Report",
-		{'report_name': report_name, 'is_standard': 'No', 'report_type': 'Custom Report'})
+	docname = frappe.db.exists(
+		"Report",
+		{
+			"report_name": report_name,
+			"is_standard": "No",
+			"report_type": "Custom Report",
+		},
+	)
 	if docname:
 		report = frappe.get_doc("Report", docname)
 		report.update({"json": columns})
@@ -500,15 +565,17 @@ def save_report(reference_report, report_name, columns):
 
 		return docname
 	else:
-		new_report = frappe.get_doc({
-			'doctype': 'Report',
-			'report_name': report_name,
-			'json': columns,
-			'ref_doctype': report_doc.ref_doctype,
-			'is_standard': 'No',
-			'report_type': 'Custom Report',
-			'reference_report': reference_report
-		}).insert(ignore_permissions = True)
+		new_report = frappe.get_doc(
+			{
+				"doctype": "Report",
+				"report_name": report_name,
+				"json": columns,
+				"ref_doctype": report_doc.ref_doctype,
+				"is_standard": "No",
+				"report_type": "Custom Report",
+				"reference_report": reference_report,
+			}
+		).insert(ignore_permissions=True)
 		frappe.msgprint(_("{0} saved successfully").format(new_report.name))
 		return new_report.name
 
@@ -526,10 +593,22 @@ def get_filtered_data(ref_doctype, columns, data, user):
 	if match_filters_per_doctype:
 		for row in data:
 			# Why linked_doctypes.get(ref_doctype)? because if column is empty, linked_doctypes[ref_doctype] is removed
-			if linked_doctypes.get(ref_doctype) and shared and row[linked_doctypes[ref_doctype]] in shared:
+			if (
+				linked_doctypes.get(ref_doctype)
+				and shared
+				and row[linked_doctypes[ref_doctype]] in shared
+			):
 				result.append(row)
 
-			elif has_match(row, linked_doctypes, match_filters_per_doctype, ref_doctype, if_owner, columns_dict, user):
+			elif has_match(
+				row,
+				linked_doctypes,
+				match_filters_per_doctype,
+				ref_doctype,
+				if_owner,
+				columns_dict,
+				user,
+			):
 				result.append(row)
 	else:
 		result = list(data)
@@ -537,17 +616,25 @@ def get_filtered_data(ref_doctype, columns, data, user):
 	return result
 
 
-def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner, columns_dict, user):
+def has_match(
+	row,
+	linked_doctypes,
+	doctype_match_filters,
+	ref_doctype,
+	if_owner,
+	columns_dict,
+	user,
+):
 	"""Returns True if after evaluating permissions for each linked doctype
-		- There is an owner match for the ref_doctype
-		- `and` There is a user permission match for all linked doctypes
+	- There is an owner match for the ref_doctype
+	- `and` There is a user permission match for all linked doctypes
 
-		Returns True if the row is empty
+	Returns True if the row is empty
 
-		Note:
-		Each doctype could have multiple conflicting user permission doctypes.
-		Hence even if one of the sets allows a match, it is true.
-		This behavior is equivalent to the trickling of user permissions of linked doctypes to the ref doctype.
+	Note:
+	Each doctype could have multiple conflicting user permission doctypes.
+	Hence even if one of the sets allows a match, it is true.
+	This behavior is equivalent to the trickling of user permissions of linked doctypes to the ref doctype.
 	"""
 	resultant_match = True
 
@@ -558,20 +645,22 @@ def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner
 	for doctype, filter_list in doctype_match_filters.items():
 		matched_for_doctype = False
 
-		if doctype==ref_doctype and if_owner:
+		if doctype == ref_doctype and if_owner:
 			idx = linked_doctypes.get("User")
-			if (idx is not None
-				and row[idx]==user
-				and columns_dict[idx]==columns_dict.get("owner")):
-					# owner match is true
-					matched_for_doctype = True
+			if (
+				idx is not None
+				and row[idx] == user
+				and columns_dict[idx] == columns_dict.get("owner")
+			):
+				# owner match is true
+				matched_for_doctype = True
 
 		if not matched_for_doctype:
 			for match_filters in filter_list:
 				match = True
 				for dt, idx in linked_doctypes.items():
 					# case handled above
-					if dt=="User" and columns_dict[idx]==columns_dict.get("owner"):
+					if dt == "User" and columns_dict[idx] == columns_dict.get("owner"):
 						continue
 
 					cell_value = None
@@ -580,7 +669,11 @@ def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner
 					elif isinstance(row, (list, tuple)):
 						cell_value = row[idx]
 
-					if dt in match_filters and cell_value not in match_filters.get(dt) and frappe.db.exists(dt, cell_value):
+					if (
+						dt in match_filters
+						and cell_value not in match_filters.get(dt)
+						and frappe.db.exists(dt, cell_value)
+					):
 						match = False
 						break
 
@@ -599,6 +692,7 @@ def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner
 
 	return resultant_match
 
+
 def get_linked_doctypes(columns, data):
 	linked_doctypes = {}
 
@@ -606,7 +700,7 @@ def get_linked_doctypes(columns, data):
 
 	for idx, col in enumerate(columns):
 		df = columns_dict[idx]
-		if df.get("fieldtype")=="Link":
+		if df.get("fieldtype") == "Link":
 			if data and isinstance(data[0], (list, tuple)):
 				linked_doctypes[df["options"]] = idx
 			else:
@@ -635,10 +729,11 @@ def get_linked_doctypes(columns, data):
 
 	return linked_doctypes
 
+
 def get_columns_dict(columns):
 	"""Returns a dict with column docfield values as dict
-		The keys for the dict are both idx and fieldname,
-		so either index or fieldname can be used to search for a column's docfield properties
+	The keys for the dict are both idx and fieldname,
+	so either index or fieldname can be used to search for a column's docfield properties
 	"""
 	columns_dict = frappe._dict()
 	for idx, col in enumerate(columns):
@@ -648,6 +743,7 @@ def get_columns_dict(columns):
 
 	return columns_dict
 
+
 def get_column_as_dict(col):
 	col_dict = frappe._dict()
 
@@ -671,6 +767,7 @@ def get_column_as_dict(col):
 
 	return col_dict
 
+
 def get_user_match_filters(doctypes, user):
 	match_filters = {}
 

frappe/email/doctype/email_domain/email_domain.py

@@ -17,6 +17,8 @@ class EmailDomain(Document):
 
 	def validate(self):
 		"""Validate email id and check POP3/IMAP and SMTP connections is enabled."""
+		logger = frappe.logger()
+
 		if self.email_id:
 			validate_email_address(self.email_id, True)
 
@@ -26,19 +28,25 @@ class EmailDomain(Document):
 		if not frappe.local.flags.in_install and not frappe.local.flags.in_patch:
 			try:
 				if self.use_imap:
+					logger.info('Checking incoming IMAP email server {host}:{port} ssl={ssl}...'.format(
+						host=self.email_server, port=get_port(self), ssl=self.use_ssl))
 					if self.use_ssl:
 						test = imaplib.IMAP4_SSL(self.email_server, port=get_port(self))
 					else:
 						test = imaplib.IMAP4(self.email_server, port=get_port(self))
 
 				else:
+					logger.info('Checking incoming POP3 email server {host}:{port} ssl={ssl}...'.format(
+						host=self.email_server, port=get_port(self), ssl=self.use_ssl))
 					if self.use_ssl:
 						test = poplib.POP3_SSL(self.email_server, port=get_port(self))
 					else:
 						test = poplib.POP3(self.email_server, port=get_port(self))
 
-			except Exception:
-				frappe.throw(_("Incoming email account not correct"))
+			except Exception as e:
+				logger.warn('Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e)
+				frappe.throw(title=_("Incoming email account not correct"),
+					msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e))
 
 			finally:
 				try:
@@ -54,22 +62,28 @@ class EmailDomain(Document):
 					if not self.get('smtp_port'):
 						self.smtp_port = 465
 
+					logger.info('Checking outgoing SMTPS email server {host}:{port}...'.format(
+						host=self.smtp_server, port=self.smtp_port))
 					sess = smtplib.SMTP_SSL((self.smtp_server or "").encode('utf-8'),
 							cint(self.smtp_port) or None)
 				else:
 					if self.use_tls and not self.smtp_port:
 						self.smtp_port = 587
+					logger.info('Checking outgoing SMTP email server {host}:{port} STARTTLS={tls}...'.format(
+						host=self.smtp_server, port=self.get('smtp_port'), tls=self.use_tls))
 					sess = smtplib.SMTP(cstr(self.smtp_server or ""), cint(self.smtp_port) or None)
 				sess.quit()
-			except Exception:
-				frappe.throw(_("Outgoing email account not correct"))
+			except Exception as e:
+				logger.warn('Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e)
+				frappe.throw(title=_("Outgoing email account not correct"),
+					msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e))
 
 	def on_update(self):
 		"""update all email accounts using this domain"""
 		for email_account in frappe.get_all("Email Account", filters={"domain": self.name}):
 			try:
 				email_account = frappe.get_doc("Email Account", email_account.name)
-				for attr in ["email_server", "use_imap", "use_ssl", "use_tls", "attachment_limit", "smtp_server", "smtp_port", "use_ssl_for_outgoing", "append_emails_to_sent_folder"]:
+				for attr in ["email_server", "use_imap", "use_ssl", "use_tls", "attachment_limit", "smtp_server", "smtp_port", "use_ssl_for_outgoing", "append_emails_to_sent_folder", "incoming_port"]:
 					email_account.set(attr, self.get(attr, default=0))
 				email_account.save()
 

frappe/email/doctype/email_domain/test_email_domain.py

@@ -5,8 +5,35 @@ from __future__ import unicode_literals
 
 import frappe
 import unittest
+from frappe.test_runner import make_test_objects
 
-# test_records = frappe.get_test_records('Domain')
+test_records = frappe.get_test_records('Email Domain')
 
 class TestDomain(unittest.TestCase):
-	pass
+
+	def setUp(self):
+		make_test_objects('Email Domain', reset=True)
+
+	def tearDown(self):
+		frappe.delete_doc("Email Account", "Test")
+		frappe.delete_doc("Email Domain", "test.com")
+
+	def test_on_update(self):
+		mail_domain = frappe.get_doc("Email Domain", "test.com")
+		mail_account = frappe.get_doc("Email Account", "Test")
+
+		# Initially, incoming_port is different in domain and account
+		self.assertNotEqual(mail_account.incoming_port, mail_domain.incoming_port)
+		# Trigger update of accounts using this domain
+		mail_domain.on_update()
+		mail_account = frappe.get_doc("Email Account", "Test")
+		# After update, incoming_port in account should match the domain
+		self.assertEqual(mail_account.incoming_port, mail_domain.incoming_port)
+
+		# Also make sure that the other attributes match
+		self.assertEqual(mail_account.use_imap, mail_domain.use_imap)
+		self.assertEqual(mail_account.use_ssl, mail_domain.use_ssl)
+		self.assertEqual(mail_account.use_tls, mail_domain.use_tls)
+		self.assertEqual(mail_account.attachment_limit, mail_domain.attachment_limit)
+		self.assertEqual(mail_account.smtp_server, mail_domain.smtp_server)
+		self.assertEqual(mail_account.smtp_port, mail_domain.smtp_port)

frappe/email/doctype/email_domain/test_records.json

@@ -0,0 +1,30 @@
+[
+ {
+  "doctype": "Email Domain",
+  "domain_name": "test.com",
+  "email_id": "_test@test.com",
+  "email_server": "imap.test.com",
+  "use_imap": "imap.test.com",
+  "use_ssl": 1,
+  "use_tls": 1,
+  "incoming_port": "993",
+  "attachment_limit": "1",
+  "smtp_server": "smtp.test.com",
+  "smtp_port": "587"
+ },
+ {
+  "doctype": "Email Account",
+  "name": "_Test Email Account 1",
+  "enable_incoming": 1,
+  "email_id": "_test@test.com",
+  "domain": "test.com",
+  "email_server": "imap.test.com",
+  "use_imap": 1,
+  "use_ssl": 0,
+  "use_tls": 1,
+  "incoming_port": "143",
+  "attachment_limit": "1",
+  "smtp_server": "smtp.test.com",
+  "smtp_port": "587"
+ }
+]

frappe/email/doctype/notification/notification.py

@@ -155,7 +155,12 @@ def get_context(context):
 				allow_update = False
 			try:
 				if allow_update and not doc.flags.in_notification_update:
-					doc.set(self.set_property_after_alert, self.property_value)
+					fieldname = self.set_property_after_alert
+					value = self.property_value
+					if doc.meta.get_field(fieldname).fieldtype in frappe.model.numeric_fieldtypes:
+						value = frappe.utils.cint(value)
+
+					doc.set(fieldname, value)
 					doc.flags.updater_reference = {
 						'doctype': self.doctype,
 						'docname': self.name,
@@ -177,7 +182,7 @@ def get_context(context):
 		recipients, cc, bcc = self.get_list_of_recipients(doc, context)
 
 		users = recipients + cc + bcc
-		
+
 		if not users:
 			return
 

frappe/email/queue.py

@@ -584,14 +584,15 @@ def prepare_message(email, recipient, recipients_list):
 
 	return safe_encode(message.as_string())
 
-def clear_outbox():
-	"""Remove low priority older than 31 days in Outbox and expire mails not sent for 7 days.
-	Called daily via scheduler.
+def clear_outbox(days=None):
+	"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
 	Note: Used separate query to avoid deadlock
 	"""
+	if not days:
+		days=31
 
 	email_queues = frappe.db.sql_list("""SELECT `name` FROM `tabEmail Queue`
-		WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '31' DAY)""")
+		WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days))
 
 	if email_queues:
 		frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format(
@@ -602,6 +603,11 @@ def clear_outbox():
 			','.join(['%s']*len(email_queues)
 		)), tuple(email_queues))
 
+def set_expiry_for_email_queue():
+	''' Mark emails as expire that has not sent for 7 days.
+		Called daily via scheduler.
+	 '''
+
 	frappe.db.sql("""
 		UPDATE `tabEmail Queue`
 		SET `status`='Expired'

frappe/geo/country_info.json

@@ -558,6 +558,8 @@
   "code": "cn",
   "currency": "CNY",
   "currency_name": "Yuan Renminbi",
+  "currency_fraction_units": 100,
+  "smallest_currency_fraction_value": 0.01,
   "date_format": "yyyy-mm-dd",
   "number_format": "#,###.##",
   "timezones": [
@@ -1389,7 +1391,10 @@
   "code": "la",
   "currency": "LAK",
   "currency_name": "Kip",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "timezones":[
+      "Asia/Vientiane"
+  ]
  },
  "Latvia": {
   "code": "lv",

frappe/hooks.py

@@ -141,6 +141,7 @@ doc_events = {
 			"frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone",
 			"frappe.core.doctype.file.file.attach_files_to_document",
 			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers",
+			"frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date",
 		],
 		"after_rename": "frappe.desk.notifications.clear_doctype_notifications",
 		"on_cancel": [
@@ -203,7 +204,7 @@ scheduler_events = {
 		"frappe.utils.password.delete_password_reset_cache"
 	],
 	"daily": [
-		"frappe.email.queue.clear_outbox",
+		"frappe.email.queue.set_expiry_for_email_queue",
 		"frappe.desk.notifications.clear_notifications",
 		"frappe.core.doctype.error_log.error_log.set_old_logs_as_seen",
 		"frappe.desk.doctype.event.event.send_event_digest",
@@ -212,7 +213,6 @@ scheduler_events = {
 		"frappe.realtime.remove_old_task_logs",
 		"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
-		"frappe.core.doctype.activity_log.activity_log.clear_authentication_logs",
 		"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",
 		"frappe.desk.form.document_follow.send_daily_updates",
 		"frappe.social.doctype.energy_point_settings.energy_point_settings.allocate_review_points",
@@ -220,7 +220,8 @@ scheduler_events = {
 		"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
 		"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
 		"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
-		"frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports"
+		"frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports",
+		"frappe.core.doctype.log_settings.log_settings.run_log_clean_up"
 	],
 	"daily_long": [
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",

frappe/installer.py

@@ -345,8 +345,7 @@ def extract_sql_gzip(sql_gz_path):
 
 	return decompressed_file
 
-
-def extract_tar_files(site_name, file_path, folder_name):
+def extract_files(site_name, file_path, folder_name):
 	import subprocess
 	import shutil
 
@@ -362,7 +361,10 @@ def extract_tar_files(site_name, file_path, folder_name):
 	tar_path = os.path.join(abs_site_path, tar_name)
 
 	try:
-		subprocess.check_output(['tar', 'xvf', tar_path, '--strip', '2'], cwd=abs_site_path)
+		if file_path.endswith(".tar"):
+			subprocess.check_output(['tar', 'xvf', tar_path, '--strip', '2'], cwd=abs_site_path)
+		elif file_path.endswith(".tgz"):
+			subprocess.check_output(['tar', 'zxvf', tar_path, '--strip', '2'], cwd=abs_site_path)
 	except:
 		raise
 	finally:

frappe/integrations/doctype/integration_request/integration_request.py

@@ -6,6 +6,8 @@ from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 import json
+from six import string_types
+from frappe.integrations.utils import json_handler
 
 class IntegrationRequest(Document):
 	def autoname(self):
@@ -20,3 +22,17 @@ class IntegrationRequest(Document):
 		self.status = status
 		self.save(ignore_permissions=True)
 		frappe.db.commit()
+
+	def handle_success(self, response):
+		"""update the output field with the response along with the relevant status"""
+		if isinstance(response, string_types):
+			response = json.loads(response)
+		self.db_set("status", "Completed")
+		self.db_set("output", json.dumps(response, default=json_handler))
+
+	def handle_failure(self, response):
+		"""update the error field with the response along with the relevant status"""
+		if isinstance(response, string_types):
+			response = json.loads(response)
+		self.db_set("status", "Failed")
+		self.db_set("error", json.dumps(response, default=json_handler))

frappe/integrations/utils.py

@@ -49,16 +49,20 @@ def make_post_request(url, auth=None, headers=None, data=None):
 		frappe.log_error()
 		raise exc
 
-def create_request_log(data, integration_type, service_name, name=None):
+def create_request_log(data, integration_type, service_name, name=None, error=None):
 	if isinstance(data, string_types):
 		data = json.loads(data)
 
+	if isinstance(error, string_types):
+		error = json.loads(error)
+
 	integration_request = frappe.get_doc({
 		"doctype": "Integration Request",
 		"integration_type": integration_type,
 		"integration_request_service": service_name,
 		"reference_doctype": data.get("reference_doctype"),
 		"reference_docname": data.get("reference_docname"),
+		"error": json.dumps(error, default=json_handler),
 		"data": json.dumps(data, default=json_handler)
 	})
 

frappe/model/base_document.py

@@ -347,7 +347,7 @@ class BaseDocument(object):
 				if self.meta.autoname=="hash":
 					# hash collision? try again
 					frappe.flags.retry_count = (frappe.flags.retry_count or 0) + 1
-					if frappe.flags.retry_count > 5:
+					if frappe.flags.retry_count > 5 and not frappe.flags.in_test:
 						raise
 					self.name = None
 					self.db_insert()

frappe/model/db_query.py

@@ -58,7 +58,10 @@ class DatabaseQuery(object):
 		if fields:
 			self.fields = fields
 		else:
-			self.fields =  ["`tab{0}`.`name`".format(self.doctype)]
+			if pluck:
+				self.fields =  ["`tab{0}`.`{1}`".format(self.doctype, pluck)]
+			else:
+				self.fields =  ["`tab{0}`.`name`".format(self.doctype)]
 
 		if start: limit_start = start
 		if page_length: limit_page_length = page_length
@@ -168,8 +171,16 @@ class DatabaseQuery(object):
 
 		fields = []
 
+		# Wrapping fields with grave quotes to allow support for sql keywords
+		# TODO: Add support for wrapping fields with sql functions and distinct keyword
 		for field in self.fields:
-			if (field.strip().startswith(("`", "*")) or "(" in field):
+			stripped_field = field.strip().lower()
+			skip_wrapping = any([
+				stripped_field.startswith(("`", "*", '"', "'")),
+				"(" in stripped_field,
+				"distinct" in stripped_field,
+			])
+			if skip_wrapping:
 				fields.append(field)
 			elif "as" in field.lower().split(" "):
 				col, _, new = field.split()

frappe/modules/utils.py

@@ -90,10 +90,11 @@ def sync_customizations(app=None):
 			folder = frappe.get_app_path(app_name, module_name, 'custom')
 			if os.path.exists(folder):
 				for fname in os.listdir(folder):
-					with open(os.path.join(folder, fname), 'r') as f:
-						data = json.loads(f.read())
-					if data.get('sync_on_migrate'):
-						sync_customizations_for_doctype(data, folder)
+					if fname.endswith('.json'):
+						with open(os.path.join(folder, fname), 'r') as f:
+							data = json.loads(f.read())
+						if data.get('sync_on_migrate'):
+							sync_customizations_for_doctype(data, folder)
 
 
 def sync_customizations_for_doctype(data, folder):

frappe/patches.txt

@@ -208,7 +208,7 @@ frappe.patches.v9_1.resave_domain_settings
 frappe.patches.v9_1.revert_domain_settings
 frappe.patches.v9_1.move_feed_to_activity_log
 execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True)
-frappe.patches.v10_0.reload_countries_and_currencies # 20-10-2020
+frappe.patches.v10_0.reload_countries_and_currencies # 14-10-2020
 frappe.patches.v10_0.refactor_social_login_keys
 frappe.patches.v10_0.enable_chat_by_default_within_system_settings
 frappe.patches.v10_0.remove_custom_field_for_disabled_domain
@@ -314,4 +314,4 @@ frappe.patches.v13_0.enable_custom_script
 frappe.patches.v13_0.update_newsletter_content_type
 execute:frappe.db.set_value('Website Settings', 'Website Settings', {'navbar_template': 'Standard Navbar', 'footer_template': 'Standard Footer'})
 frappe.patches.v13_0.delete_event_producer_and_consumer_keys
-frappe.patches.v13_0.web_template_set_module
+frappe.patches.v13_0.web_template_set_module #2020-10-05

frappe/patches/v13_0/generate_theme_files_in_public_folder.py

@@ -6,6 +6,7 @@ import frappe
 
 
 def execute():
+	frappe.reload_doc("website", "doctype", "website_theme_ignore_app")
 	themes = frappe.db.get_all(
 		"Website Theme", filters={"theme_url": ("not like", "/files/website_theme/%")}
 	)

frappe/patches/v13_0/web_template_set_module.py

@@ -6,7 +6,9 @@ import frappe
 
 def execute():
 	"""Set default module for standard Web Template, if none."""
-	frappe.reload_doc('website', 'doctype', 'Web Template')
+	frappe.reload_doc('website', 'doctype', 'Web Template Field')
+	frappe.reload_doc('website', 'doctype', 'web_template')
+
 	standard_templates = frappe.get_list('Web Template', {'standard': 1})
 	for template in standard_templates:
 		doc = frappe.get_doc('Web Template', template.name)

frappe/public/js/frappe/desk.js

@@ -154,6 +154,26 @@ frappe.Application = Class.extend({
 					}
 				});
 			}, 300000); // check every 5 minutes
+
+			if(frappe.user.has_role("System Manager")){
+				setInterval(function() {
+					frappe.call({
+						method: 'frappe.core.doctype.log_settings.log_settings.has_unseen_error_log',
+						args: {
+							user: frappe.session.user
+						},
+						callback: function(r) {
+							console.log(r);
+							if(r.message.show_alert){
+								frappe.show_alert({
+									indicator: 'red',
+									message: r.message.message
+								});
+							}
+						}
+					});
+				}, 600000); // check every 10 minutes
+			}
 		}
 
 		this.fetch_tags();

frappe/public/js/frappe/form/controls/dynamic_link.js

@@ -1,14 +1,12 @@
 frappe.ui.form.ControlDynamicLink = frappe.ui.form.ControlLink.extend({
 	get_options: function() {
 		let options = '';
-		if(this.df.get_options) {
+		if (this.df.get_options) {
 			options = this.df.get_options();
-		}
-		else if (this.docname==null && cur_dialog) {
+		} else if (this.docname==null && cur_dialog) {
 			//for dialog box
 			options = cur_dialog.get_value(this.df.options);
-		}
-		else if (!cur_frm) {
+		} else if (!cur_frm) {
 			const selector = `input[data-fieldname="${this.df.options}"]`;
 			let input = null;
 			if (cur_list) {
@@ -21,13 +19,12 @@ frappe.ui.form.ControlDynamicLink = frappe.ui.form.ControlLink.extend({
 			if (input) {
 				options = input.val();
 			}
-		}
-		else {
+		} else {
 			options = frappe.model.get_value(this.df.parent, this.docname, this.df.options);
 		}
 
 		if (frappe.model.is_single(options)) {
-			frappe.throw(__(`${options.bold()} is not a valid DocType for Dynamic Link`));
+			frappe.throw(__("{0} is not a valid DocType for Dynamic Link", [options.bold()]));
 		}
 
 		return options;

frappe/public/js/frappe/form/controls/int.js

@@ -1,17 +1,17 @@
 frappe.ui.form.ControlInt = frappe.ui.form.ControlData.extend({
-	make: function() {
+	make: function () {
 		this._super();
 		// $(this.label_area).addClass('pull-right');
 		// $(this.disp_area).addClass('text-right');
 	},
-	make_input: function() {
+	make_input: function () {
 		var me = this;
 		this._super();
 		this.$input
 			// .addClass("text-right")
-			.on("focus", function() {
-				setTimeout(function() {
-					if(!document.activeElement) return;
+			.on("focus", function () {
+				setTimeout(function () {
+					if (!document.activeElement) return;
 					document.activeElement.value
 						= me.validate(document.activeElement.value);
 					document.activeElement.select();
@@ -19,7 +19,10 @@ frappe.ui.form.ControlInt = frappe.ui.form.ControlData.extend({
 				return false;
 			});
 	},
-	eval_expression: function(value) {
+	validate: function (value) {
+		return this.parse(value);
+	},
+	eval_expression: function (value) {
 		if (typeof value === 'string') {
 			if (value.match(/^[0-9+\-/* ]+$/)) {
 				// If it is a string containing operators
@@ -33,7 +36,7 @@ frappe.ui.form.ControlInt = frappe.ui.form.ControlData.extend({
 		}
 		return value;
 	},
-	parse: function(value) {
+	parse: function (value) {
 		return cint(this.eval_expression(value), null);
 	}
 });

frappe/public/js/frappe/form/form.js

@@ -1522,11 +1522,12 @@ frappe.ui.form.Form = class FrappeForm {
 
 					const escaped_name = encodeURIComponent(value);
 
-					return repl('<a class="indicator %(color)s" href="#Form/%(doctype)s/%(name)s">%(label)s</a>', {
+					return repl('<a class="indicator %(color)s" href="#Form/%(doctype)s/%(escaped_name)s" data-doctype="%(doctype)s" data-name="%(name)s">%(label)s</a>', {
 						color: get_color(doc || {}),
 						doctype: df.options,
-						name: escaped_name,
-						label: label
+						escaped_name: escaped_name,
+						label: label,
+						name: value
 					});
 				} else {
 					return '';

frappe/public/js/frappe/form/grid.js

@@ -775,6 +775,13 @@ export default class Grid {
 			// download
 			this.setup_download();
 
+			const value_formatter_map = {
+				"Date": val => val ? frappe.datetime.user_to_str(val) : val,
+				"Int": val => cint(val),
+				"Check": val => cint(val),
+				"Float": val => flt(val),
+			};
+
 			// upload
 			frappe.flags.no_socketio = true;
 			$(this.wrapper).find(".grid-upload").removeClass('hidden').on("click", () => {
@@ -802,16 +809,9 @@ export default class Grid {
 										var fieldname = fieldnames[ci];
 										var df = frappe.meta.get_docfield(me.df.options, fieldname);
 
-										// convert date formatting
-										if (df.fieldtype === "Date" && value) {
-											value = frappe.datetime.user_to_str(value);
-										}
-
-										if (df.fieldtype === "Int" || df.fieldtype === "Check") {
-											value = cint(value);
-										}
-
-										d[fieldnames[ci]] = value;
+										d[fieldnames[ci]] = value_formatter_map[df.fieldtype]
+											? value_formatter_map[df.fieldtype](value)
+											: value;
 									});
 								}
 							}

frappe/public/js/frappe/form/sidebar/review.js

@@ -4,7 +4,7 @@
 
 
 frappe.ui.form.Review = class Review {
-	constructor({parent, frm}) {
+	constructor({ parent, frm }) {
 		this.parent = parent;
 		this.frm = frm;
 		this.points = frappe.boot.points;
@@ -49,7 +49,7 @@ frappe.ui.form.Review = class Review {
 		const docinfo = this.frm.get_docinfo();
 
 		involved_users = involved_users.concat(
-			docinfo.communications.map(d => d.sender && d.delivery_status==='sent'),
+			docinfo.communications.map(d => d.sender && d.delivery_status === 'sent'),
 			docinfo.comments.map(d => d.owner),
 			docinfo.versions.map(d => d.owner),
 			docinfo.assignments.map(d => d.owner)
@@ -89,7 +89,7 @@ frappe.ui.form.Review = class Review {
 				fieldtype: 'Int',
 				label: __('Points'),
 				reqd: 1,
-				description: __(`Currently you have ${this.points.review_points} review points`)
+				description: __("Currently you have {0} review points", [this.points.review_points])
 			}, {
 				fieldtype: 'Small Text',
 				fieldname: 'reason',
@@ -132,7 +132,7 @@ frappe.ui.form.Review = class Review {
 		this.reviews.empty();
 		review_logs.forEach(log => {
 			let review_pill = $(`
-				<li class="review ${log.points < 0 ? 'criticism': 'appreciation'}">
+				<li class="review ${log.points < 0 ? 'criticism' : 'appreciation'}">
 					<div>
 						${Math.abs(log.points)}
 					</div>
@@ -166,7 +166,7 @@ frappe.ui.form.Review = class Review {
 			delay: 500,
 			placement: 'top',
 			template: `
-				<div class="popover" role="tooltip">
+				<div class="review-popover popover" role="tooltip">
 					<div class="arrow"></div>
 					<h3 class="popover-header"></h3>
 					<div class="popover-body">

frappe/public/js/frappe/form/success_action.js

@@ -28,8 +28,9 @@ frappe.ui.form.SuccessAction = class SuccessAction {
 	}
 
 	show_alert() {
-		frappe.db.count(this.form.doctype)
-			.then(count => {
+		frappe.db.get_list(this.form.doctype, {limit: 2})
+			.then(result => {
+				const count = result.length;
 				const setting = this.setting;
 				let message = count === 1 ?
 					setting.first_success_message :

frappe/public/js/frappe/list/list_view.js

@@ -36,8 +36,8 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 		this.parent.disable_scroll_to_top = true;
 
 		if (!this.has_permissions()) {
-			frappe.set_route("");
-			frappe.msgprint(__(`Not permitted to view ${this.doctype}`));
+			frappe.set_route('');
+			frappe.msgprint(__("Not permitted to view {0}", [this.doctype]));
 			return;
 		}
 

frappe/public/js/frappe/request.js

@@ -444,7 +444,7 @@ frappe.request.report_error = function(xhr, request_opts) {
 		var communication_composer = new frappe.views.CommunicationComposer({
 			subject: 'Error Report [' + frappe.datetime.nowdate() + ']',
 			recipients: error_report_email,
-			message: error_report_message,
+			message: frappe.utils.xss_sanitise(error_report_message),
 			doc: {
 				doctype: "User",
 				name: frappe.session.user

frappe/public/js/frappe/router.js

@@ -168,7 +168,14 @@ frappe.set_route = function() {
 			}
 		}).join('/');
 
-		window.location.hash = route;
+		// Perform a redirect when redirect is set in route_options
+		if (frappe.route_options && frappe.route_options.redirect) {
+			const url = new URL(window.location);
+			url.hash = route;
+			window.location.replace(url);
+		} else {
+			window.location.hash = route;
+		}
 
 		// Set favicon (app.js)
 		frappe.provide('frappe.app');

frappe/public/js/frappe/ui/capture.js

@@ -3,155 +3,139 @@
 
 /**
  * @description Converts a canvas, image or a video to a data URL string.
- * 
+ *
  * @param 	{HTMLElement} element - canvas, img or video.
  * @returns {string} 			  - The data URL string.
- * 
+ *
  * @example
  * frappe._.get_data_uri(video)
  * // returns "data:image/pngbase64,..."
  */
-frappe._.get_data_uri = element =>
-{
-	const $element = $(element)
-	const width    = $element.width()
-	const height   = $element.height()
+frappe._.get_data_uri = element => {
+	const $element = $(element);
+	const width = $element.width();
+	const height = $element.height();
 
-	const $canvas     = $('<canvas/>')
-	$canvas[0].width  = width
-	$canvas[0].height = height
+	const $canvas = $('<canvas/>');
+	$canvas[0].width = width;
+	$canvas[0].height = height;
 
-	const context     = $canvas[0].getContext('2d')
-	context.drawImage($element[0], 0, 0, width, height)
-	
-	const data_uri = $canvas[0].toDataURL('image/png')
+	const context = $canvas[0].getContext('2d');
+	context.drawImage($element[0], 0, 0, width, height);
 
-	return data_uri
-}
+	const data_uri = $canvas[0].toDataURL('image/png');
+
+	return data_uri;
+};
 
 /**
  * @description Frappe's Capture object.
- * 
+ *
  * @example
  * const capture = frappe.ui.Capture()
  * capture.show()
- * 
+ *
  * capture.click((data_uri) => {
  * 	// do stuff
  * })
- * 
+ *
  * @see https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Taking_still_photos
  */
-frappe.ui.Capture = class
-{
-	constructor (options = { })
-	{
-		this.options = frappe.ui.Capture.OPTIONS
-		this.set_options(options)
+frappe.ui.Capture = class {
+	constructor(options = {}) {
+		this.options = frappe.ui.Capture.OPTIONS;
+		this.set_options(options);
 	}
-	
-	set_options (options)
-	{
-		this.options = { ...frappe.ui.Capture.OPTIONS, ...options }
-		
-		return this
+
+	set_options(options) {
+		this.options = { ...frappe.ui.Capture.OPTIONS, ...options };
+
+		return this;
 	}
-	
-	render ( )
-	{
-		return navigator.mediaDevices.getUserMedia({ video: true }).then(stream =>
-		{
-			this.dialog 	 = new frappe.ui.Dialog({
-				  title: this.options.title,
+
+	render() {
+		return navigator.mediaDevices.getUserMedia({ video: true }).then(stream => {
+			this.dialog = new frappe.ui.Dialog({
+				title: this.options.title,
 				animate: this.options.animate,
-				 action:
-				{
-					secondary:
-					{
-						label: "<b>&times</b>"
+				action: {
+					secondary: {
+						label: '<b>&times</b>'
 					}
 				}
-			})
-	
-			const $e 		 = $(frappe.ui.Capture.TEMPLATE)
-			
-			const video      = $e.find('video')[0]
-			video.srcObject  = stream
-			video.play()
-			
-			const $container = $(this.dialog.body)
-			$container.html($e)
-			
-			$e.find('.fc-btf').hide()
+			});
 
-			$e.find('.fc-bcp').click(() =>
-			{
-				const data_url = frappe._.get_data_uri(video)
-				$e.find('.fc-p').attr('src', data_url)
+			const $e = $(frappe.ui.Capture.TEMPLATE);
 
-				$e.find('.fc-s').hide()
-				$e.find('.fc-p').show()
+			const video = $e.find('video')[0];
+			video.srcObject = stream;
+			video.play();
 
-				$e.find('.fc-btu').hide()
-				$e.find('.fc-btf').show()
-			})
+			const $container = $(this.dialog.body);
+			$container.html($e);
 
-			$e.find('.fc-br').click(() =>
-			{
-				$e.find('.fc-p').hide()
-				$e.find('.fc-s').show()
+			$e.find('.fc-btf').hide();
 
-				$e.find('.fc-btf').hide()
-				$e.find('.fc-btu').show()
-			})
+			$e.find('.fc-bcp').click(() => {
+				const data_url = frappe._.get_data_uri(video);
+				$e.find('.fc-p').attr('src', data_url);
 
-			$e.find('.fc-bs').click(() =>
-			{
-				const data_url = frappe._.get_data_uri(video)
-				this.hide()
-				
-				if (this.callback)
-					this.callback(data_url)
-			})
-		})
+				$e.find('.fc-s').hide();
+				$e.find('.fc-p').show();
+
+				$e.find('.fc-btu').hide();
+				$e.find('.fc-btf').show();
+			});
+
+			$e.find('.fc-br').click(() => {
+				$e.find('.fc-p').hide();
+				$e.find('.fc-s').show();
+
+				$e.find('.fc-btf').hide();
+				$e.find('.fc-btu').show();
+			});
+
+			$e.find('.fc-bs').click(() => {
+				const data_url = frappe._.get_data_uri(video);
+				this.hide();
+
+				if (this.callback) this.callback(data_url);
+			});
+		});
 	}
 
-	show ( )
-	{
-		this.render().then(() =>
-		{
-			this.dialog.show()
-		}).catch(err => {
-			if ( this.options.error )
-			{
-				const alert = `<span class="indicator red"/> ${frappe.ui.Capture.ERR_MESSAGE}`
-				frappe.show_alert(alert, 3)
-			}
+	show() {
+		this.render()
+			.then(() => {
+				this.dialog.show();
+			})
+			.catch(err => {
+				if (this.options.error) {
+					const alert = `<span class="indicator red"/> ${
+						frappe.ui.Capture.ERR_MESSAGE
+					}`;
+					frappe.show_alert(alert, 3);
+				}
 
-			throw err
-		})
+				throw err;
+			});
 	}
 
-	hide ( )
-	{
-		if ( this.dialog )
-			this.dialog.hide()
+	hide() {
+		if (this.dialog) this.dialog.hide();
 	}
 
-	submit (fn)
-	{
-		this.callback = fn
+	submit(fn) {
+		this.callback = fn;
 	}
-}
-frappe.ui.Capture.OPTIONS =
-{
-	  title: __(`Camera`),
+};
+frappe.ui.Capture.OPTIONS = {
+	title: __("Camera"),
 	animate: false,
-	  error: false,
-}
-frappe.ui.Capture.ERR_MESSAGE = __("Unable to load camera.")
-frappe.ui.Capture.TEMPLATE 	  =
-`
+	error: false
+};
+frappe.ui.Capture.ERR_MESSAGE = __('Unable to load camera.');
+frappe.ui.Capture.TEMPLATE = `
 <div class="frappe-capture">
 	<div class="panel panel-default">
 		<img class="fc-p img-responsive"/>
@@ -181,14 +165,7 @@ frappe.ui.Capture.TEMPLATE 	  =
 		<div class="fc-btu">
 			<div class="row">
 				<div class="col-md-6">
-					${
-						''
-						// <div class="pull-left">
-						// 	<button class="btn btn-default">
-						// 		<small>${__('Take Video')}</small>
-						// 	</button>
-						// </div>
-					}
+					${''}
 				</div>
 				<div class="col-md-6">
 					<div class="pull-right">
@@ -201,4 +178,4 @@ frappe.ui.Capture.TEMPLATE 	  =
 		</div>
 	</div>
 </div>
-`
+`;

frappe/public/js/frappe/ui/filters/filter.js

@@ -54,7 +54,7 @@ frappe.ui.Filter = class {
 			this.filters_config = frappe.boot.additional_filters_config;
 			for (let key of Object.keys(this.filters_config)) {
 				const filter = this.filters_config[key];
-				this.conditions.push([key, __(`{0}`, [filter.label])]);
+				this.conditions.push([key, __(filter.label)]);
 				for (let fieldtype of Object.keys(this.invalid_condition_map)) {
 					if (!filter.valid_for_fieldtypes.includes(fieldtype)) {
 						this.invalid_condition_map[fieldtype].push(key);
@@ -543,13 +543,13 @@ frappe.ui.filter_utils = {
 			if (period_map[period]) {
 				period_map[period].forEach((p) => {
 					options.push({
-						label: __(`{0} {1}`, [period, p]),
+						label: `${period} ${p}`,
 						value: `${period.toLowerCase()} ${p.toLowerCase()}`,
 					});
 				});
 			} else {
 				options.push({
-					label: __(`{0}`, [period]),
+					label: __(period),
 					value: `${period.toLowerCase()}`,
 				});
 			}

frappe/public/js/frappe/ui/filters/filter_list.js

@@ -164,13 +164,11 @@ frappe.ui.FilterGroup = class {
 	}
 
 	validate_args(doctype, fieldname) {
-		if (
-			doctype &&
-			fieldname &&
-			!frappe.meta.has_field(doctype, fieldname) &&
-			!frappe.model.std_fields_list.includes(fieldname)
-		) {
-			frappe.throw(__(`Invalid filter: "${[fieldname.bold()]}"`));
+		if (doctype && fieldname
+			&& !frappe.meta.has_field(doctype, fieldname)
+			&& !frappe.model.std_fields_list.includes(fieldname)) {
+
+			frappe.throw(__("Invalid filter: {0}", [fieldname.bold()]));
 			return false;
 		}
 		return true;

frappe/public/js/frappe/utils/dashboard_utils.js

@@ -195,7 +195,7 @@ frappe.dashboard_utils = {
 				try {
 					f[3] = eval(f[3]);
 				} catch (e) {
-					frappe.throw(__(`Invalid expression set in filter ${f[1]} (${f[0]})`));
+					frappe.throw(__("Invalid expression set in filter {0} ({1})", [f[1], f[0]]));
 				}
 			});
 			filters = [...filters, ...dynamic_filters];
@@ -205,7 +205,7 @@ frappe.dashboard_utils = {
 					const val = eval(dynamic_filters[key]);
 					dynamic_filters[key] = val;
 				} catch (e) {
-					frappe.throw(__(`Invalid expression set in filter ${key}`));
+					frappe.throw(__("Invalid expression set in filter {0}", [key]));
 				}
 			}
 			Object.assign(filters, dynamic_filters);
@@ -251,7 +251,7 @@ frappe.dashboard_utils = {
 					let dashboard_route_html =
 						`<a href = "#dashboard/${values.dashboard}">${values.dashboard}</a>`;
 					let message =
-						__(`${doctype} ${values.name} added to Dashboard ` + dashboard_route_html);
+						__("{0} {1} added to Dashboard {2}", [doctype, values.name, dashboard_route_html]);
 
 					frappe.msgprint(message);
 				});

frappe/public/js/frappe/views/dashboard/dashboard_view.js

@@ -271,7 +271,7 @@ frappe.views.DashboardView = class DashboardView extends frappe.views.ListView {
 	show_add_chart_dialog() {
 		let fields = this.get_field_options();
 		const dialog = new frappe.ui.Dialog({
-			title: __(`Add a ${this.doctype} Chart`),
+			title: __("Add a {0} Chart", [this.doctype]),
 			fields: [
 				{
 					fieldname: 'new_or_existing',

frappe/public/js/frappe/views/reports/query_report.js

@@ -335,12 +335,12 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			let message;
 			if (dashboard_name) {
 				let dashboard_route_html = `<a href = "#dashboard/${dashboard_name}">${dashboard_name}</a>`;
-				message = __(`New {0} {1} added to Dashboard ` + dashboard_route_html, [doctype, name]);
+				message = __("New {0} {1} added to Dashboard {2}", [doctype, name, dashboard_route_html]);
 			} else {
-				message = __(`New {0} {1} created`, [doctype, name]);
+				message = __("New {0} {1} created", [doctype, name]);
 			}
 
-			frappe.msgprint(message, __(`New {0} Created`, [doctype]));
+			frappe.msgprint(message, __("New {0} Created", [doctype]));
 		});
 	}
 
@@ -447,7 +447,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				try {
 					out = eval(expression.substr(5));
 				} catch (e) {
-					frappe.throw(__(`Invalid "depends_on" expression set in filter ${filter_label}`));
+					frappe.throw(__('Invalid "depends_on" expression set in filter {0}', [filter_label]));
 				}
 			} else {
 				var value = doc[expression];
@@ -483,7 +483,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			df.onchange = () => {
 				this.refresh_filters_dependency();
 
-				let current_filters = this.get_filter_value();
+				let current_filters = this.get_filter_values();
 				if (this.previous_filters
 					&& (JSON.stringify(this.previous_filters) === JSON.stringify(current_filters))) {
 					// filter values have not changed
@@ -755,14 +755,18 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 
 	get_queued_prepared_reports_warning_message(reports) {
 		const route = `#List/Prepared Report/List?status=Queued&report_name=${this.report_name}`;
+		const report_link_html = reports.length == 1
+			? `<a class="underline" href="${route}">${__('1 Report')}</a>`
+			: `<a class="underline" href="${route}">${__("{0} Reports", [reports.length])}</a>`;
+
 		const no_of_reports_html = reports.length == 1
-			? `${__('There is ')}<a class="underline" href="${route}">${__('1 Report')}</a>`
-			: `${__('There are ')}<a class="underline" href="${route}">${__(`{} Reports`, [reports.length])}</a>`;
+			? `${__('There is {0} with the same filters already in the queue:', [report_link_html])}`
+			: `${__('There are {0} with the same filters already in the queue:', [report_link_html])}`;
 
 		let warning_message = `
 			<p>
-				${__(`Are you sure you want to generate a new report?
-					{} with the same filters already in the queue:`, [no_of_reports_html])}
+				${__("Are you sure you want to generate a new report?")}
+				${no_of_reports_html}
 			</p>`;
 
 		let get_item_html = item => `<a class="underline" href="#Form/Prepared Report/${item.name}">${item.name}</a>`;
@@ -1031,7 +1035,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 						field.value == values.x_field
 					)[0].label;
 
-				options.title = __(`${this.report_name}: ${x_field_label} vs ${y_field_label}`);
+				options.title = __("{0}: {1} vs {2}", [this.report_name, x_field_label, y_field_label]);
 
 				this.render_chart(options);
 				this.add_chart_buttons_to_toolbar(true);
@@ -1502,7 +1506,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 								insert_after_index: insert_after_index,
 								link_field: this.doctype_field_map[values.doctype],
 								doctype: values.doctype,
-								options: df.fieldtype === "Link" ? df.options : undefined,
+								options: df.options,
 								width: 100
 							});
 

frappe/public/js/frappe/views/reports/report_view.js

@@ -650,6 +650,9 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 	}
 
 	set_fields() {
+		// default fields
+		['name', 'docstatus'].map((f) => this._add_field(f));
+
 		if (this.report_name && this.report_doc.json.fields) {
 			let fields = this.report_doc.json.fields.slice();
 			fields.forEach(f => this._add_field(f[0], f[1]));
@@ -666,12 +669,11 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 
 	set_default_fields() {
 		// get fields from meta
-		this.fields = [];
+		this.fields = this.fields || [];
 		const add_field = f => this._add_field(f);
 
 		// default fields
 		[
-			'name', 'docstatus',
 			this.meta.title_field,
 			this.meta.image_field
 		].map(add_field);

frappe/public/js/frappe/views/workspace/workspace.js

@@ -295,6 +295,7 @@ class DesktopPage {
 	}
 
 	save_customization() {
+		frappe.dom.freeze();
 		const config = {};
 
 		if (this.sections.charts) config.charts = this.sections.charts.get_widget_config();
@@ -305,6 +306,7 @@ class DesktopPage {
 			page: this.page_name,
 			config: config
 		}).then(res => {
+			frappe.dom.unfreeze();
 			if (res.message) {
 				frappe.show_alert({ message: __("Customizations Saved Successfully"), indicator: "green" });
 				this.reload();
@@ -312,7 +314,7 @@ class DesktopPage {
 				frappe.throw({ message: __("Something went wrong while saving customizations"), indicator: "red" });
 				this.reload();
 			}
-		});
+		})
 	}
 
 	reset_customization() {
@@ -326,7 +328,7 @@ class DesktopPage {
 
 	make_onboarding() {
 		this.onboarding_widget = frappe.widget.make_widget({
-			label: this.data.onboarding.label || __(`Let's Get Started`),
+			label: this.data.onboarding.label || __("Let's Get Started"),
 			subtitle: this.data.onboarding.subtitle,
 			steps: this.data.onboarding.items,
 			success: this.data.onboarding.success,
@@ -392,7 +394,7 @@ class DesktopPage {
 
 	make_cards() {
 		let cards = new frappe.widget.WidgetGroup({
-			title: this.data.cards.label || __(`Reports & Masters`),
+			title: this.data.cards.label || __("Reports & Masters"),
 			container: this.page,
 			type: "links",
 			columns: 3,

frappe/public/js/frappe/widgets/chart_widget.js

@@ -407,7 +407,7 @@ export default class ChartWidget extends Widget {
 	setup_filter_dialog(fields) {
 		let me = this;
 		let dialog = new frappe.ui.Dialog({
-			title: __(`Set Filters for ${this.chart_doc.chart_name}`),
+			title: __("Set Filters for {0}", [this.chart_doc.chart_name]),
 			fields: fields,
 			primary_action: function() {
 				let values = this.get_values();

frappe/public/js/frappe/widgets/new_widget.js

@@ -19,7 +19,8 @@ export default class NewWidget {
 	get_title() {
 		// DO NOT REMOVE: Comment to load translation
 		// __("New Chart") __("New Shortcut") __("New Number Card")
-		return __(`New ${frappe.model.unscrub(this.type)}`);
+		let title = `New ${frappe.model.unscrub(this.type)}`;
+		return __(title);
 	}
 
 	make_widget() {

frappe/public/js/frappe/widgets/onboarding_widget.js

@@ -27,7 +27,7 @@ export default class OnboardingWidget extends Widget {
 
 		let $step = $(`<a class="onboarding-step ${status}">
 				<div class="step-title">
-					<div class="step-index step-pending">${ __(index + 1) }</div>
+					<div class="step-index step-pending">${__(index + 1)}</div>
 					<div class="step-index step-skipped">${frappe.utils.icon('tick', 'xs')}</div>
 					<div class="step-index step-complete">${frappe.utils.icon('tick', 'xs')}</div>
 					<div>${step.title}</div>
@@ -83,7 +83,11 @@ export default class OnboardingWidget extends Widget {
 			this.step_body.empty();
 			this.step_footer.empty();
 
-			this.step_body.html(frappe.markdown(step.description) || `<h1>${step.title}</h1>`)
+			this.step_body.html(
+				step.description ?
+					frappe.markdown(step.description)
+					: `<h1>${step.title}</h1>`
+			)
 
 			if (step.intro_video_url) {
 				$(`<button class="btn btn-primary btn-sm">${__('Watch Tutorial')}</button>`)
@@ -110,10 +114,15 @@ export default class OnboardingWidget extends Widget {
 			$(`<button class="btn btn-primary btn-sm">${__(step.action)}</button>`)
 				.appendTo(this.step_footer)
 				.on('click', () => {
-					plyr.pause()
-					actions[step.action](step)
+					plyr.pause();
+					actions[step.action](step);
 				});
 
+			// Fire only once, on hashchange
+			$(window).one('hashchange', () => {
+				plyr.pause();
+			})
+
 			$(`<button class="btn btn-secondary ml-2 btn-sm">${__('Back')}</button>`)
 				.appendTo(this.step_footer)
 				.on('click', toggle_content);

frappe/public/js/frappe/widgets/widget_dialog.js

@@ -35,7 +35,8 @@ class WidgetDialog {
 		// __("New Chart") __("New Shortcut") __("Edit Chart") __("Edit Shortcut")
 
 		let action = this.editing ? "Edit" : "Add";
-		return __(`${action} ${frappe.model.unscrub(this.type)}`);
+		let label = action = action + " " + frappe.model.unscrub(this.type);
+		return __(label);
 	}
 
 	get_fields() {

frappe/social/doctype/energy_point_log/energy_point_log.json

@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "creation": "2018-06-21 14:58:55.913619",
  "doctype": "DocType",
  "editable_grid": 1,
@@ -109,8 +110,9 @@
    "label": "Seen"
   }
  ],
- "in_create": 1,
- "modified": "2019-08-21 15:51:05.288886",
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2020-10-06 17:25:40.477044",
  "modified_by": "Administrator",
  "module": "Social",
  "name": "Energy Point Log",

frappe/templates/includes/app_analytics/google_analytics.html

@@ -6,6 +6,9 @@
   })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
 
   ga('create', '{{ google_analytics_id }}', 'auto');
+  {% if google_analytics_anonymize_ip %}
+  ga('set', 'anonymizeIp', true);
+  {% endif %}
 
   $(document).on("mousedown", function(event) {
     if(!frappe && !frappe.get_route) return;

frappe/templates/includes/footer/footer_links.html

@@ -1,5 +1,5 @@
 {% macro footer_link(item) %}
-<a href="{{ item.url | abs_url }}" class="footer-link">
+<a href="{{ item.url | abs_url }}" {{ item.target }} class="footer-link">
 	{%- if item.icon -%}
 	<img src="{{ item.icon }}" alt="{{ item.label }}">
 	{%- else -%}

frappe/tests/test_commands.py

@@ -1,12 +1,15 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 
 # imports - standard imports
+import os
 import shlex
 import subprocess
 import unittest
+from glob import glob
 
 # imports - module imports
 import frappe
+from frappe.utils.backups import fetch_latest_backups
 
 
 def clean(value):
@@ -15,9 +18,14 @@ def clean(value):
 	return value
 
 
-class BaseTestCommands:
-	def execute(self, command):
-		command = command.format(**{"site": frappe.local.site})
+class BaseTestCommands(unittest.TestCase):
+	def execute(self, command, kwargs=None):
+		site = {"site": frappe.local.site}
+		if kwargs:
+			kwargs.update(site)
+		else:
+			kwargs = site
+		command = command.replace("\n", " ").format(**kwargs)
 		command = shlex.split(command)
 		self._proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 		self.stdout = clean(self._proc.stdout)
@@ -25,7 +33,7 @@ class BaseTestCommands:
 		self.returncode = clean(self._proc.returncode)
 
 
-class TestCommands(BaseTestCommands, unittest.TestCase):
+class TestCommands(BaseTestCommands):
 	def test_execute(self):
 		# test 1: execute a command expecting a numeric output
 		self.execute("bench --site {site} execute frappe.db.get_database_size")
@@ -44,3 +52,70 @@ class TestCommands(BaseTestCommands, unittest.TestCase):
 		self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""")
 		self.assertEquals(self.returncode, 0)
 		self.assertEquals(self.stdout[1:-1], frappe.bold(text='DocType'))
+
+	def test_backup(self):
+		home = os.path.expanduser("~")
+		site_backup_path = frappe.utils.get_site_path("private", "backups")
+
+		# test 1: take a backup
+		before_backup = fetch_latest_backups()
+		self.execute("bench --site {site} backup")
+		after_backup = fetch_latest_backups()
+
+		self.assertEquals(self.returncode, 0)
+		self.assertIn("successfully completed", self.stdout)
+		self.assertNotEqual(before_backup["database"], after_backup["database"])
+
+		# test 2: take a backup with --with-files
+		before_backup = after_backup.copy()
+		self.execute("bench --site {site} backup --with-files")
+		after_backup = fetch_latest_backups()
+
+		self.assertEquals(self.returncode, 0)
+		self.assertIn("successfully completed", self.stdout)
+		self.assertIn("with files", self.stdout)
+		self.assertNotEqual(before_backup, after_backup)
+		self.assertIsNotNone(after_backup["public"])
+		self.assertIsNotNone(after_backup["private"])
+
+		# test 3: take a backup with --backup-path
+		backup_path = os.path.join(home, "backups")
+		self.execute("bench --site {site} backup --backup-path {backup_path}", {"backup_path": backup_path})
+
+		self.assertEquals(self.returncode, 0)
+		self.assertTrue(os.path.exists(backup_path))
+		self.assertGreaterEqual(len(os.listdir(backup_path)), 2)
+
+		# test 4: take a backup with --backup-path-db, --backup-path-files, --backup-path-private-files, --backup-path-conf
+		kwargs = {
+			key: os.path.join(home, key, value)
+			for key, value in {
+				"db_path": "database.sql.gz",
+				"files_path": "public.tar",
+				"private_path": "private.tar",
+				"conf_path": "config.json"
+			}.items()
+		}
+
+		self.execute("""bench
+			--site {site} backup --with-files
+			--backup-path-db {db_path}
+			--backup-path-files {files_path}
+			--backup-path-private-files {private_path}
+			--backup-path-conf {conf_path}""", kwargs)
+
+		self.assertEquals(self.returncode, 0)
+		for path in kwargs.values():
+			self.assertTrue(os.path.exists(path))
+
+		# test 5: take a backup with --compress
+		self.execute("bench --site {site} backup --with-files --compress")
+
+		self.assertEquals(self.returncode, 0)
+
+		compressed_files = glob(site_backup_path + "/*.tgz")
+		self.assertGreater(len(compressed_files), 0)
+
+		# test 6: take a backup with --verbose
+		self.execute("bench --site {site} backup --verbose")
+		self.assertEquals(self.returncode, 0)

frappe/tests/test_db.py

@@ -133,6 +133,27 @@ class TestDB(unittest.TestCase):
 		self.assertEqual(list(frappe.get_all("ToDo", fields=[random_field], limit=1)[0])[0], random_field)
 		self.assertEqual(list(frappe.get_all("ToDo", fields=["{0} as total".format(random_field)], limit=1)[0])[0], "total")
 
+		# Testing read for distinct and sql functions
+		self.assertEqual(list(
+			frappe.get_all("ToDo",
+				fields=["`{0}` as total".format(random_field)],
+				distinct=True,
+				limit=1,
+			)[0]
+		)[0], "total")
+		self.assertEqual(list(
+			frappe.get_all("ToDo",
+			fields=["`{0}`".format(random_field)],
+			distinct=True,
+			limit=1,
+			)[0]
+		)[0], random_field)
+		self.assertEqual(list(
+			frappe.get_all("ToDo",
+				fields=["count(`{0}`)".format(random_field)],
+				limit=1
+			)[0]
+		)[0], "count" if frappe.conf.db_type == "postgres" else "count(`{0}`)".format(random_field))
 
 		# Testing update
 		frappe.db.set_value(test_doctype, random_doc, random_field, random_value)

frappe/tests/test_db_query.py

@@ -347,6 +347,14 @@ class TestReportview(unittest.TestCase):
 			limit=50,
 		)
 
+	def test_pluck_name(self):
+		names = DatabaseQuery("DocType").execute(filters={"name": "DocType"}, pluck="name")
+		self.assertEqual(names, ["DocType"])
+
+	def test_pluck_any_field(self):
+		owners = DatabaseQuery("DocType").execute(filters={"name": "DocType"}, pluck="owner")
+		self.assertEqual(owners, ["Administrator"])
+
 def create_event(subject="_Test Event", starts_on=None):
 	""" create a test event """
 

frappe/tests/test_email.py

@@ -130,8 +130,10 @@ class TestEmail(unittest.TestCase):
 	def test_expired(self):
 		self.test_email_queue()
 		frappe.db.sql("UPDATE `tabEmail Queue` SET `modified`=(NOW() - INTERVAL '8' day)")
-		from frappe.email.queue import clear_outbox
-		clear_outbox()
+
+		from frappe.email.queue import set_expiry_for_email_queue
+		set_expiry_for_email_queue()
+
 		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Expired'""", as_dict=1)
 		self.assertEqual(len(email_queue), 1)
 		queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient`

frappe/tests/test_scheduler.py

@@ -31,7 +31,7 @@ class TestScheduler(TestCase):
 		enqueue_events(site = frappe.local.site)
 		frappe.flags.execute_job = False
 
-		self.assertTrue('frappe.email.queue.clear_outbox', frappe.flags.enqueued_jobs)
+		self.assertTrue('frappe.email.queue.set_expiry_for_email_queue', frappe.flags.enqueued_jobs)
 		self.assertTrue('frappe.utils.change_log.check_for_update', frappe.flags.enqueued_jobs)
 		self.assertTrue('frappe.email.doctype.auto_email_report.auto_email_report.send_monthly', frappe.flags.enqueued_jobs)
 

frappe/translate.py

@@ -620,7 +620,7 @@ def get_untranslated(lang, untranslated_file, get_all=False):
 
 	if get_all:
 		print(str(len(messages)) + " messages")
-		with open(untranslated_file, "w") as f:
+		with open(untranslated_file, "wb") as f:
 			for m in messages:
 				# replace \n with ||| so that internal linebreaks don't get split
 				f.write((escape_newlines(m[1]) + os.linesep).encode("utf-8"))
@@ -633,10 +633,10 @@ def get_untranslated(lang, untranslated_file, get_all=False):
 
 		if untranslated:
 			print(str(len(untranslated)) + " missing translations of " + str(len(messages)))
-			with open(untranslated_file, "w") as f:
+			with open(untranslated_file, "wb") as f:
 				for m in untranslated:
 					# replace \n with ||| so that internal linebreaks don't get split
-					f.write(cstr(frappe.safe_encode(escape_newlines(m) + os.linesep)))
+					f.write((escape_newlines(m) + os.linesep).encode("utf-8"))
 		else:
 			print("all translated!")