Merge remote-tracking branch 'upstream/develop' into esbuild

.eslintrc

@@ -143,6 +143,7 @@
 		"Cypress": true,
 		"cy": true,
 		"it": true,
+		"describe": true,
 		"expect": true,
 		"context": true,
 		"before": true,

.github/helper/semgrep_rules/frappe_correctness.py

@@ -0,0 +1,28 @@
+import frappe
+from frappe import _, flt
+
+from frappe.model.document import Document
+
+
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	# ruleid: frappe-modifying-after-submit
+	self.status = 'Submitted'
+
+def on_submit(self):   # noqa
+	if flt(self.per_billed) < 100:
+		self.update_billing_status()
+	else:
+		# todook: frappe-modifying-after-submit
+		self.status = "Completed"
+		self.db_set("status", "Completed")
+
+class TestDoc(Document):
+	pass
+
+	def validate(self):
+		#ruleid: frappe-modifying-child-tables-while-iterating
+		for item in self.child_table:
+			if item.value < 0:
+				self.remove(item)

.github/helper/semgrep_rules/frappe_correctness.yml

@@ -0,0 +1,135 @@
+# This file specifies rules for correctness according to how frappe doctype data model works.
+
+rules:
+- id: frappe-modifying-but-not-comitting
+  patterns:
+    - pattern: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.save()
+    - metavariable-regex:
+        metavariable: '$ATTR'
+        # this is negative look-ahead, add more attrs to ignore like (ignore|ignore_this_too|ignore_me)
+        regex: '^(?!ignore_linked_doctypes|status_updater)(.*)$'
+    - metavariable-regex:
+        metavariable: "$METHOD"
+        regex: "(on_submit|on_cancel)"
+  message: |
+    DocType modified in self.$METHOD. Please check if modification of self.$ATTR is commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-modifying-but-not-comitting-other-method
+  patterns:
+  - pattern: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+          self.save()
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - metavariable-regex:
+      metavariable: "$METHOD"
+      regex: "(on_submit|on_cancel)"
+  message: |
+    self.$ANOTHER_METHOD is called from self.$METHOD, check if changes to self.$ATTR are commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-print-function-in-doctypes
+  pattern: print(...)
+  message: |
+      Did you mean to leave this print statement in? Consider using msgprint or logger instead of print statement.
+  languages: [python]
+  severity: WARNING
+  paths:
+      exclude:
+        - test_*.py
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-modifying-child-tables-while-iterating
+  pattern-either:
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.remove(...)
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.append(...)
+  message: |
+      Child table being modified while iterating on it.
+  languages: [python]
+  severity: ERROR
+  paths:
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-same-key-assigned-twice
+  pattern-either:
+    - pattern: |
+        {..., $X: $A, ..., $X: $B, ...}
+    - pattern: |
+        dict(..., ($X, $A), ..., ($X, $B), ...)
+    - pattern: |
+        _dict(..., ($X, $A), ..., ($X, $B), ...)
+  message: |
+      key `$X` is uselessly assigned twice. This could be a potential bug.
+  languages: [python]
+  severity: ERROR

.github/helper/semgrep_rules/security.yml

@@ -12,3 +12,18 @@ rules:
     exclude:
       - frappe/__init__.py
       - frappe/commands/utils.py
+
+- id: frappe-sqli-format-strings
+  patterns:
+    - pattern-inside: |
+        @frappe.whitelist()
+        def $FUNC(...):
+            ...
+    - pattern-either:
+        - pattern: frappe.db.sql("..." % ...)
+        - pattern: frappe.db.sql(f"...", ...)
+        - pattern: frappe.db.sql("...".format(...), ...)
+  message: |
+      Detected use of raw string formatting for SQL queries. This can lead to sql injection vulnerabilities. Refer security guidelines - https://github.com/frappe/erpnext/wiki/Code-Security-Guidelines
+  languages: [python]
+  severity: WARNING

.github/helper/semgrep_rules/translate.yml

@@ -44,7 +44,8 @@ rules:
   pattern-either:
   - pattern: _(...) + ... + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'
+  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations

.github/helper/semgrep_rules/ux.py

@@ -0,0 +1,31 @@
+import frappe
+from frappe import msgprint, throw, _
+
+
+# ruleid: frappe-missing-translate-function
+throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.msgprint("Useful message")
+
+# ruleid: frappe-missing-translate-function
+msgprint("Useful message")
+
+
+# ok: frappe-missing-translate-function
+translatedmessage = _("Hello")
+
+# ok: frappe-missing-translate-function
+throw(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(_("Helpful message"))
+
+# ok: frappe-missing-translate-function
+frappe.throw(_("Error occured"))

.github/helper/semgrep_rules/ux.yml

@@ -0,0 +1,15 @@
+rules:
+- id: frappe-missing-translate-function
+  pattern-either:
+  - patterns:
+      - pattern: frappe.msgprint("...", ...)
+      - pattern-not: frappe.msgprint(_("..."), ...)
+      - pattern-not: frappe.msgprint(__("..."), ...)
+  - patterns:
+      - pattern: frappe.throw("...", ...)
+      - pattern-not: frappe.throw(_("..."), ...)
+      - pattern-not: frappe.throw(__("..."), ...)
+  message: |
+      All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
+  languages: [python, javascript, json]
+  severity: ERROR

.github/workflows/ci-tests.yml

@@ -151,7 +151,8 @@ jobs:
           cd ${GITHUB_WORKSPACE}
           pip install coveralls==3.0.1
           pip install coverage==5.5
-          coveralls --service=github-actions
+          coveralls --service=github
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+          COVERALLS_SERVICE_NAME: github

.github/workflows/semgrep.yml

@@ -14,9 +14,19 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: Run semgrep
+
+    - name: Setup semgrep
       run: |
         python -m pip install -q semgrep
         git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
+
+    - name: Semgrep errors
+      run: |
         files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files
+        semgrep --config="r/python.lang.correctness" --quiet --error $files
+
+    - name: Semgrep warnings
+      run: |
+        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files

README.md

@@ -39,7 +39,8 @@ Full-stack web application framework that uses Python and MariaDB on the server
 
 ### Installation
 
-[Install via Frappe Bench](https://github.com/frappe/bench)
+* [Install via Docker](https://github.com/frappe/frappe_docker)
+* [Install via Frappe Bench](https://github.com/frappe/bench)
 
 ## Contributing
 

frappe/api.py

@@ -1,12 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-from __future__ import unicode_literals
 
 import base64
 import binascii
 import json
-
-from six.moves.urllib.parse import urlencode, urlparse
+from urllib.parse import urlencode, urlparse
 
 import frappe
 import frappe.client
@@ -14,6 +12,7 @@ import frappe.handler
 from frappe import _
 from frappe.utils.response import build_response
 
+
 def handle():
 	"""
 	Handler for `/api` methods
@@ -38,9 +37,6 @@ def handle():
 	`/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method
 	"""
 
-
-	validate_auth()
-
 	parts = frappe.request.path[1:].split("/",3)
 	call = doctype = name = None
 
@@ -116,7 +112,7 @@ def handle():
 						frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields'])
 					frappe.local.form_dict.setdefault('limit_page_length', 20)
 					frappe.local.response.update({
-						"data":  frappe.call(
+						"data": frappe.call(
 							frappe.client.get_list,
 							doctype,
 							**frappe.local.form_dict
@@ -140,6 +136,7 @@ def handle():
 
 	return build_response("json")
 
+
 def get_request_form_data():
 	if frappe.local.form_dict.data is None:
 		data = frappe.safe_decode(frappe.local.request.get_data())
@@ -148,25 +145,18 @@ def get_request_form_data():
 
 	return frappe.parse_json(data)
 
+
 def validate_auth():
-	if frappe.get_request_header("Authorization") is None:
-		return
-
-	VALID_AUTH_PREFIX_TYPES = ['basic', 'bearer', 'token']
-	VALID_AUTH_PREFIX_STRING = ", ".join(VALID_AUTH_PREFIX_TYPES).title()
-
+	"""
+	Authenticate and sets user for the request.
+	"""
 	authorization_header = frappe.get_request_header("Authorization", str()).split(" ")
-	authorization_type = authorization_header[0].lower()
 
-	if len(authorization_header) == 1:
-		frappe.throw(_('Invalid Authorization headers, add a token with a prefix from one of the following: {0}.').format(VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationHeader)
-
-	if authorization_type == "bearer":
+	if len(authorization_header) == 2:
 		validate_oauth(authorization_header)
-	elif authorization_type in VALID_AUTH_PREFIX_TYPES:
 		validate_auth_via_api_keys(authorization_header)
-	else:
-		frappe.throw(_('Invalid Authorization Type {0}, must be one of {1}.').format(authorization_type, VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationPrefix)
+
+	validate_auth_via_hooks()
 
 
 def validate_oauth(authorization_header):
@@ -177,8 +167,8 @@ def validate_oauth(authorization_header):
 		authorization_header (list of str): The 'Authorization' header containing the prefix and token
 	"""
 
-	from frappe.oauth import get_url_delimiter
 	from frappe.integrations.oauth2 import get_oauth_server
+	from frappe.oauth import get_url_delimiter
 
 	form_dict = frappe.local.form_dict
 	token = authorization_header[1]
@@ -192,14 +182,13 @@ def validate_oauth(authorization_header):
 
 	try:
 		required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter())
+		valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes)
+		if valid:
+			frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
+			frappe.local.form_dict = form_dict
 	except AttributeError:
-		frappe.throw(_("Invalid Bearer token, please provide a valid access token with prefix 'Bearer'."), frappe.InvalidAuthorizationToken)
+		pass
 
-	valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes)
-
-	if valid:
-		frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
-		frappe.local.form_dict = form_dict
 
 
 def validate_auth_via_api_keys(authorization_header):
@@ -222,8 +211,7 @@ def validate_auth_via_api_keys(authorization_header):
 	except binascii.Error:
 		frappe.throw(_("Failed to decode token, please provide a valid base64-encoded token."), frappe.InvalidAuthorizationToken)
 	except (AttributeError, TypeError, ValueError):
-		frappe.throw(_("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'."), frappe.InvalidAuthorizationToken)
-
+		pass
 
 
 def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None):
@@ -248,3 +236,8 @@ def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=Non
 		if frappe.local.login_manager.user in ('', 'Guest'):
 			frappe.set_user(user)
 		frappe.local.form_dict = form_dict
+
+
+def validate_auth_via_hooks():
+	for auth_hook in frappe.get_hooks('auth_hooks', []):
+		frappe.get_attr(auth_hook)()

frappe/app.py

@@ -56,6 +56,7 @@ def application(request):
 		frappe.recorder.record()
 		frappe.monitor.start()
 		frappe.rate_limiter.apply()
+		frappe.api.validate_auth()
 
 		if request.method == "OPTIONS":
 			response = Response()

frappe/commands/__init__.py

@@ -62,11 +62,24 @@ def popen(command, *args, **kwargs):
 	if env:
 		env = dict(environ, **env)
 
+	def set_low_prio():
+		import psutil
+		if psutil.LINUX:
+			psutil.Process().nice(19)
+			psutil.Process().ionice(psutil.IOPRIO_CLASS_IDLE)
+		elif psutil.WINDOWS:
+			psutil.Process().nice(psutil.IDLE_PRIORITY_CLASS)
+			psutil.Process().ionice(psutil.IOPRIO_VERYLOW)
+		else:
+			psutil.Process().nice(19)
+			# ionice not supported
+
 	proc = subprocess.Popen(command,
 		stdout=None if output else subprocess.PIPE,
 		stderr=None if output else subprocess.PIPE,
 		shell=shell,
 		cwd=cwd,
+		preexec_fn=set_low_prio,
 		env=env
 	)
 

frappe/commands/scheduler.py

@@ -18,22 +18,33 @@ def _is_scheduler_enabled():
 
 	return enable_scheduler
 
-@click.command('trigger-scheduler-event')
-@click.argument('event')
+
+@click.command("trigger-scheduler-event", help="Trigger a scheduler event")
+@click.argument("event")
 @pass_context
 def trigger_scheduler_event(context, event):
-	"Trigger a scheduler event"
 	import frappe.utils.scheduler
+
+	exit_code = 0
+
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			frappe.utils.scheduler.trigger(site, event, now=True)
+			try:
+				frappe.get_doc("Scheduled Job Type", {"method": event}).execute()
+			except frappe.DoesNotExistError:
+				click.secho(f"Event {event} does not exist!", fg="red")
+				exit_code = 1
 		finally:
 			frappe.destroy()
+
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
+	sys.exit(exit_code)
+
+
 @click.command('enable-scheduler')
 @pass_context
 def enable_scheduler(context):

frappe/email/doctype/newsletter/newsletter.py

@@ -24,6 +24,7 @@ class Newsletter(WebsiteGenerator):
 		if self.send_from:
 			validate_email_address(self.send_from, True)
 
+	@frappe.whitelist()
 	def test_send(self, doctype="Lead"):
 		self.recipients = frappe.utils.split_emails(self.test_email_id)
 		self.queue_all(test_email=True)

frappe/handler.py

@@ -9,7 +9,6 @@ import frappe
 import frappe.utils
 import frappe.sessions
 from frappe.utils import cint
-from frappe.api import validate_auth
 from frappe import _, is_whitelisted
 from frappe.utils.response import build_response
 from frappe.utils.csvutils import build_csv_response
@@ -24,7 +23,7 @@ ALLOWED_MIMETYPES = ('image/png', 'image/jpeg', 'application/pdf', 'application/
 
 def handle():
 	"""handle request"""
-	validate_auth()
+
 	cmd = frappe.local.form_dict.cmd
 	data = None
 

frappe/hooks.py

@@ -132,6 +132,16 @@ has_website_permission = {
 	"Address": "frappe.contacts.doctype.address.address.has_website_permission"
 }
 
+jinja = {
+	"methods": "frappe.utils.jinja_globals",
+	"filters": [
+		"frappe.utils.data.global_date_format",
+		"frappe.utils.markdown",
+		"frappe.website.utils.get_shade",
+		"frappe.website.utils.abs_url",
+	]
+}
+
 standard_queries = {
 	"User": "frappe.core.doctype.user.user.user_query"
 }

frappe/patches.txt

@@ -335,3 +335,4 @@ frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings
 frappe.patches.v13_0.remove_twilio_settings
 frappe.patches.v12_0.rename_uploaded_files_with_proper_name
 frappe.patches.v13_0.queryreport_columns
+frappe.patches.v13_0.jinja_hook

frappe/patches/v13_0/jinja_hook.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+from click import secho
+
+def execute():
+	if frappe.get_hooks('jenv'):
+		print()
+		secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow')
+		secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow')
+		print()

frappe/public/js/frappe/form/controls/base_control.js

@@ -159,9 +159,13 @@ frappe.ui.form.Control = class BaseControl {
 	}
 	validate_and_set_in_model(value, e) {
 		var me = this;
-		if(this.inside_change_event) {
+		let force_value_set = (this.doc && this.doc.__run_link_triggers);
+		let is_value_same = (this.get_model_value() === value);
+
+		if (this.inside_change_event || (!force_value_set && is_value_same)) {
 			return Promise.resolve();
 		}
+
 		this.inside_change_event = true;
 		var set = function(value) {
 			me.inside_change_event = false;

frappe/public/js/frappe/form/controls/table_multiselect.js

@@ -66,6 +66,10 @@ frappe.ui.form.ControlTableMultiSelect = class ControlTableMultiSelect extends f
 		this._rows_list = this.rows.map(row => row[link_field.fieldname]);
 		return this.rows;
 	}
+	get_model_value() {
+		let value = this._super();
+		return value ? value.filter(d => !d.__islocal) : value;
+	}
 	validate(value) {
 		const rows = (value || []).slice();
 

frappe/public/js/frappe/form/form.js

@@ -1203,8 +1203,7 @@ frappe.ui.form.Form = class FrappeForm {
 
 		$.each(grid_field_label_map, function(fname, label) {
 			fname = fname.split("-");
-			var df = frappe.meta.get_docfield(fname[0], fname[1], me.doc.name);
-			if(df) df.label = label;
+			me.fields_dict[parentfield].grid.update_docfield_property(fname[1], 'label', label);
 		});
 	}
 

frappe/public/js/frappe/form/grid.js

@@ -387,6 +387,8 @@ export default class Grid {
 			this.wrapper.find('.grid-footer').toggle(false);
 		}
 
+		this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable());
+
 	}
 
 	truncate_rows() {

frappe/public/js/frappe/form/grid_row.js

@@ -557,13 +557,10 @@ export default class GridRow {
 		this.row.toggle(false);
 		// this.form_panel.toggle(true);
 
-		if (this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows)) {
-			this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row')
-				.addClass('hidden');
-		} else {
-			this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row')
-				.removeClass('hidden');
-		}
+		let cannot_add_rows = this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows);
+		this.wrapper
+			.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row')
+			.toggle(!cannot_add_rows);
 
 		frappe.dom.freeze("", "dark");
 		if (cur_frm) cur_frm.cur_grid = this;

frappe/public/js/frappe/form/grid_row_form.js

@@ -119,7 +119,7 @@ export default class GridRowForm {
 		});
 	}
 	toggle_add_delete_button_display($parent) {
-		$parent.find(".row-actions")
+		$parent.find(".row-actions, .grid-append-row")
 			.toggle(this.row.grid.is_editable());
 	}
 	refresh_field(fieldname) {

frappe/templates/base.html

@@ -60,7 +60,7 @@
 		window.is_chat_enabled = {{ chat_enable }};
 	</script>
 </head>
-<body frappe-session-status="{{ 'logged-in' if frappe.session.user != 'Guest' else 'logged-out'}}" data-path="{{ path | e }}" {%- if template and template.endswith('.md') %} frappe-content-type="markdown" {% endif -%} class="{{ body_class or ''}}">
+<body frappe-session-status="{{ 'logged-in' if frappe.session.user != 'Guest' else 'logged-out'}}" data-path="{{ path | e }}" {%- if template and template.endswith('.md') %} frappe-content-type="markdown" {%- endif %} class="{{ body_class or ''}}">
 	{% include "public/icons/timeless/symbol-defs.svg" %}
 	{%- block banner -%}
 		{% include "templates/includes/banner_extension.html" ignore missing %}

frappe/utils/__init__.py

@@ -307,14 +307,23 @@ def unesc(s, esc_chars):
 		s = s.replace(esc_str, c)
 	return s
 
-def execute_in_shell(cmd, verbose=0):
+def execute_in_shell(cmd, verbose=0, low_priority=False):
 	# using Popen instead of os.system - as recommended by python docs
 	import tempfile
 	from subprocess import Popen
 
 	with tempfile.TemporaryFile() as stdout:
 		with tempfile.TemporaryFile() as stderr:
-			p = Popen(cmd, shell=True, stdout=stdout, stderr=stderr)
+			kwargs = {
+				"shell": True,
+				"stdout": stdout,
+				"stderr": stderr
+			}
+
+			if low_priority:
+				kwargs["preexec_fn"] = lambda: os.nice(10)
+
+			p = Popen(cmd, **kwargs)
 			p.wait()
 
 			stdout.seek(0)

frappe/utils/backups.py

@@ -315,8 +315,6 @@ class BackupGenerator:
 			print(template.format(_type.title(), info["path"], info["size"]))
 
 	def backup_files(self):
-		import subprocess
-
 		for folder in ("public", "private"):
 			files_path = frappe.get_site_path(folder, "files")
 			backup_path = (
@@ -327,12 +325,12 @@ class BackupGenerator:
 				cmd_string = "tar cf - {1} | gzip > {0}"
 			else:
 				cmd_string = "tar -cf {0} {1}"
-			output = subprocess.check_output(
-				cmd_string.format(backup_path, files_path), shell=True
-			)
 
-			if self.verbose and output:
-				print(output.decode("utf8"))
+			frappe.utils.execute_in_shell(
+				cmd_string.format(backup_path, files_path),
+				verbose=self.verbose,
+				low_priority=True
+			)
 
 	def copy_site_config(self):
 		site_config_backup_path = self.backup_path_conf
@@ -436,7 +434,7 @@ class BackupGenerator:
 		if self.verbose:
 			print(command + "\n")
 
-		err, out = frappe.utils.execute_in_shell(command)
+		frappe.utils.execute_in_shell(command, low_priority=True)
 
 	def send_email(self):
 		"""

frappe/utils/boilerplate.py

@@ -190,6 +190,15 @@ app_license = "{app_license}"
 # automatically create page for each record of this doctype
 # website_generators = ["Web Page"]
 
+# Jinja
+# ----------
+
+# add methods and filters to jinja environment
+# jinja = {{
+# 	"methods": "{app_name}.utils.jinja_methods",
+# 	"filters": "{app_name}.utils.jinja_filters"
+# }}
+
 # Installation
 # ------------
 
@@ -303,6 +312,13 @@ user_data_fields = [
 	}}
 ]
 
+# Authentication and authorization
+# --------------------------------
+
+# auth_hooks = [
+# 	"{app_name}.auth.validate"
+# ]
+
 """
 
 desktop_template = """# -*- coding: utf-8 -*-

frappe/utils/jinja.py

@@ -18,16 +18,10 @@ def get_jenv():
 		set_filters(jenv)
 
 		jenv.globals.update(get_safe_globals())
-		jenv.globals.update(get_jenv_customization('methods'))
-		jenv.globals.update({
-			'resolve_class': resolve_class,
-			'inspect': inspect,
-			'web_blocks': web_blocks,
-			'web_block': web_block,
-			'script': script,
-			'style': style,
-			'assets_url': assets_url
-		})
+
+		methods, filters = get_jinja_hooks()
+		jenv.globals.update(methods or {})
+		jenv.filters.update(filters or {})
 
 		frappe.local.jenv = jenv
 
@@ -130,125 +124,49 @@ def get_jloader():
 
 def set_filters(jenv):
 	import frappe
-	from frappe.utils import global_date_format, cint, cstr, flt, markdown
-	from frappe.website.utils import get_shade, abs_url
+	from frappe.utils import cint, cstr, flt
 
-	jenv.filters["global_date_format"] = global_date_format
-	jenv.filters["markdown"] = markdown
 	jenv.filters["json"] = frappe.as_json
-	jenv.filters["get_shade"] = get_shade
 	jenv.filters["len"] = len
 	jenv.filters["int"] = cint
 	jenv.filters["str"] = cstr
 	jenv.filters["flt"] = flt
-	jenv.filters["abs_url"] = abs_url
 
 	if frappe.flags.in_setup_help:
 		return
 
-	jenv.filters.update(get_jenv_customization('filters'))
-
-
-def get_jenv_customization(customization_type):
-	'''Returns a dict with filter/method name as key and definition as value'''
 
+def get_jinja_hooks():
+	"""Returns a tuple of (methods, filters) each containing a dict of method name and method definition pair."""
 	import frappe
 
-	out = {}
 	if not getattr(frappe.local, "site", None):
+		return (None, None)
+
+	from types import FunctionType, ModuleType
+	from inspect import getmembers, isfunction
+
+	def get_obj_dict_from_paths(object_paths):
+		out = {}
+		for obj_path in object_paths:
+			try:
+				obj = frappe.get_module(obj_path)
+			except ModuleNotFoundError:
+				obj = frappe.get_attr(obj_path)
+
+			if isinstance(obj, ModuleType):
+				functions = getmembers(obj, isfunction)
+				for function_name, function in functions:
+					out[function_name] = function
+			elif isinstance(obj, FunctionType):
+				function_name = obj.__name__
+				out[function_name] = obj
 		return out
 
-	values = frappe.get_hooks("jenv", {}).get(customization_type)
-	if not values:
-		return out
+	values = frappe.get_hooks("jinja")
+	methods, filters = values.get("methods", []), values.get("filters", [])
 
-	for value in values:
-		fn_name, fn_string = value.split(":")
-		out[fn_name] = frappe.get_attr(fn_string)
+	method_dict = get_obj_dict_from_paths(methods)
+	filter_dict = get_obj_dict_from_paths(filters)
 
-	return out
-
-
-def resolve_class(classes):
-	import frappe
-
-	if classes is None:
-		return ''
-
-	if isinstance(classes, frappe.string_types):
-		return classes
-
-	if isinstance(classes, (list, tuple)):
-		return ' '.join([resolve_class(c) for c in classes]).strip()
-
-	if isinstance(classes, dict):
-		return ' '.join([classname for classname in classes if classes[classname]]).strip()
-
-	return classes
-
-
-def inspect(var, render=True):
-	context = { "var": var }
-	if render:
-		html = "<pre>{{ var | pprint | e }}</pre>"
-	else:
-		html = ""
-	return get_jenv().from_string(html).render(context)
-
-
-def web_block(template, values=None, **kwargs):
-	options = {"template": template, "values": values}
-	options.update(kwargs)
-	return web_blocks([options])
-
-
-def web_blocks(blocks):
-	from frappe import throw, _dict
-	from frappe.website.doctype.web_page.web_page import get_web_blocks_html
-
-	web_blocks = []
-	for block in blocks:
-		if not block.get('template'):
-			throw('Web Template is not specified')
-
-		doc = _dict({
-			'doctype': 'Web Page Block',
-			'web_template': block['template'],
-			'web_template_values': block.get('values', {}),
-			'add_top_padding': 1,
-			'add_bottom_padding': 1,
-			'add_container': 1,
-			'hide_block': 0,
-			'css_class': ''
-		})
-		doc.update(block)
-		web_blocks.append(doc)
-
-	out = get_web_blocks_html(web_blocks)
-
-	html = out.html
-	for script in out.scripts:
-		html += '<script>{}</script>'.format(script)
-
-	return html
-
-def script(path):
-	path = assets_url(path)
-	if '/public/' in path:
-		path = path.replace('/public/', '/dist/')
-	return f'<script type="text/javascript" src="{path}"></script>'
-
-def style(path):
-	path = assets_url(path)
-	if '/public/' in path:
-		path = path.replace('/public/', '/dist/')
-	if path.endswith(('.scss', '.sass', '.less', '.styl')):
-		path = path.rsplit('.', 1)[0] + '.css'
-	return f'<link type="text/css" rel="stylesheet" href="{path}">'
-
-def assets_url(path):
-	if not path.startswith('/'):
-		path = '/' + path
-	if not path.startswith('/assets'):
-		path = '/assets' + path
-	return path
+	return method_dict, filter_dict

frappe/utils/jinja_globals.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+from frappe.utils.jinja import get_jenv
+import frappe
+
+
+def resolve_class(classes):
+	if classes is None:
+		return ""
+
+	if isinstance(classes, frappe.string_types):
+		return classes
+
+	if isinstance(classes, (list, tuple)):
+		return " ".join([resolve_class(c) for c in classes]).strip()
+
+	if isinstance(classes, dict):
+		return " ".join([classname for classname in classes if classes[classname]]).strip()
+
+	return classes
+
+
+def inspect(var, render=True):
+	context = {"var": var}
+	if render:
+		html = "<pre>{{ var | pprint | e }}</pre>"
+	else:
+		return ""
+	return get_jenv().from_string(html).render(context)
+
+
+def web_block(template, values=None, **kwargs):
+	options = {"template": template, "values": values}
+	options.update(kwargs)
+	return web_blocks([options])
+
+
+def web_blocks(blocks):
+	from frappe import throw, _dict
+	from frappe.website.doctype.web_page.web_page import get_web_blocks_html
+
+	web_blocks = []
+	for block in blocks:
+		if not block.get("template"):
+			throw("Web Template is not specified")
+
+		doc = _dict(
+			{
+				"doctype": "Web Page Block",
+				"web_template": block["template"],
+				"web_template_values": block.get("values", {}),
+				"add_top_padding": 1,
+				"add_bottom_padding": 1,
+				"add_container": 1,
+				"hide_block": 0,
+				"css_class": "",
+			}
+		)
+		doc.update(block)
+		web_blocks.append(doc)
+
+	out = get_web_blocks_html(web_blocks)
+
+	html = out.html
+	for script in out.scripts:
+		html += "<script>{}</script>".format(script)
+
+	return html
+
+def script(path):
+	path = assets_url(path)
+	if '/public/' in path:
+		path = path.replace('/public/', '/dist/')
+	return f'<script type="text/javascript" src="{path}"></script>'
+
+def style(path):
+	path = assets_url(path)
+	if '/public/' in path:
+		path = path.replace('/public/', '/dist/')
+	if path.endswith(('.scss', '.sass', '.less', '.styl')):
+		path = path.rsplit('.', 1)[0] + '.css'
+	return f'<link type="text/css" rel="stylesheet" href="{path}">'
+
+def assets_url(path):
+	if not path.startswith('/'):
+		path = '/' + path
+	if not path.startswith('/assets'):
+		path = '/assets' + path
+	return path

frappe/website/doctype/web_page/web_page.py

@@ -242,11 +242,11 @@ def extract_script_and_style_tags(html):
 	styles = []
 
 	for script in soup.find_all('script'):
-		scripts.append(script.text)
+		scripts.append(script.string)
 		script.extract()
 
 	for style in soup.find_all('style'):
-		styles.append(style.text)
+		styles.append(style.string)
 		style.extract()
 
 	return str(soup), scripts, styles