Merge branch 'develop' of https://github.com/frappe/frappe into refactor-website

.eslintrc

@@ -143,6 +143,7 @@
 		"Cypress": true,
 		"cy": true,
 		"it": true,
+		"describe": true,
 		"expect": true,
 		"context": true,
 		"before": true,

.github/helper/semgrep_rules/frappe_correctness.py

@@ -0,0 +1,28 @@
+import frappe
+from frappe import _, flt
+
+from frappe.model.document import Document
+
+
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	# ruleid: frappe-modifying-after-submit
+	self.status = 'Submitted'
+
+def on_submit(self):   # noqa
+	if flt(self.per_billed) < 100:
+		self.update_billing_status()
+	else:
+		# todook: frappe-modifying-after-submit
+		self.status = "Completed"
+		self.db_set("status", "Completed")
+
+class TestDoc(Document):
+	pass
+
+	def validate(self):
+		#ruleid: frappe-modifying-child-tables-while-iterating
+		for item in self.child_table:
+			if item.value < 0:
+				self.remove(item)

.github/helper/semgrep_rules/frappe_correctness.yml

@@ -0,0 +1,135 @@
+# This file specifies rules for correctness according to how frappe doctype data model works.
+
+rules:
+- id: frappe-modifying-but-not-comitting
+  patterns:
+    - pattern: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.save()
+    - metavariable-regex:
+        metavariable: '$ATTR'
+        # this is negative look-ahead, add more attrs to ignore like (ignore|ignore_this_too|ignore_me)
+        regex: '^(?!ignore_linked_doctypes|status_updater)(.*)$'
+    - metavariable-regex:
+        metavariable: "$METHOD"
+        regex: "(on_submit|on_cancel)"
+  message: |
+    DocType modified in self.$METHOD. Please check if modification of self.$ATTR is commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-modifying-but-not-comitting-other-method
+  patterns:
+  - pattern: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+          self.save()
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - metavariable-regex:
+      metavariable: "$METHOD"
+      regex: "(on_submit|on_cancel)"
+  message: |
+    self.$ANOTHER_METHOD is called from self.$METHOD, check if changes to self.$ATTR are commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-print-function-in-doctypes
+  pattern: print(...)
+  message: |
+      Did you mean to leave this print statement in? Consider using msgprint or logger instead of print statement.
+  languages: [python]
+  severity: WARNING
+  paths:
+      exclude:
+        - test_*.py
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-modifying-child-tables-while-iterating
+  pattern-either:
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.remove(...)
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.append(...)
+  message: |
+      Child table being modified while iterating on it.
+  languages: [python]
+  severity: ERROR
+  paths:
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-same-key-assigned-twice
+  pattern-either:
+    - pattern: |
+        {..., $X: $A, ..., $X: $B, ...}
+    - pattern: |
+        dict(..., ($X, $A), ..., ($X, $B), ...)
+    - pattern: |
+        _dict(..., ($X, $A), ..., ($X, $B), ...)
+  message: |
+      key `$X` is uselessly assigned twice. This could be a potential bug.
+  languages: [python]
+  severity: ERROR

.github/helper/semgrep_rules/security.yml

@@ -12,3 +12,18 @@ rules:
     exclude:
       - frappe/__init__.py
       - frappe/commands/utils.py
+
+- id: frappe-sqli-format-strings
+  patterns:
+    - pattern-inside: |
+        @frappe.whitelist()
+        def $FUNC(...):
+            ...
+    - pattern-either:
+        - pattern: frappe.db.sql("..." % ...)
+        - pattern: frappe.db.sql(f"...", ...)
+        - pattern: frappe.db.sql("...".format(...), ...)
+  message: |
+      Detected use of raw string formatting for SQL queries. This can lead to sql injection vulnerabilities. Refer security guidelines - https://github.com/frappe/erpnext/wiki/Code-Security-Guidelines
+  languages: [python]
+  severity: WARNING

.github/helper/semgrep_rules/translate.yml

@@ -44,7 +44,8 @@ rules:
   pattern-either:
   - pattern: _(...) + ... + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'
+  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations

.github/helper/semgrep_rules/ux.py

@@ -0,0 +1,31 @@
+import frappe
+from frappe import msgprint, throw, _
+
+
+# ruleid: frappe-missing-translate-function
+throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.msgprint("Useful message")
+
+# ruleid: frappe-missing-translate-function
+msgprint("Useful message")
+
+
+# ok: frappe-missing-translate-function
+translatedmessage = _("Hello")
+
+# ok: frappe-missing-translate-function
+throw(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(_("Helpful message"))
+
+# ok: frappe-missing-translate-function
+frappe.throw(_("Error occured"))

.github/helper/semgrep_rules/ux.yml

@@ -0,0 +1,15 @@
+rules:
+- id: frappe-missing-translate-function
+  pattern-either:
+  - patterns:
+      - pattern: frappe.msgprint("...", ...)
+      - pattern-not: frappe.msgprint(_("..."), ...)
+      - pattern-not: frappe.msgprint(__("..."), ...)
+  - patterns:
+      - pattern: frappe.throw("...", ...)
+      - pattern-not: frappe.throw(_("..."), ...)
+      - pattern-not: frappe.throw(__("..."), ...)
+  message: |
+      All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
+  languages: [python, javascript, json]
+  severity: ERROR

.github/workflows/ci-tests.yml

@@ -149,9 +149,10 @@ jobs:
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls
+          pip install coveralls==3.0.1
+          pip install coverage==5.5
+          coveralls --service=github
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+          COVERALLS_SERVICE_NAME: github

.github/workflows/semgrep.yml

@@ -14,9 +14,19 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: Run semgrep
+
+    - name: Setup semgrep
       run: |
         python -m pip install -q semgrep
         git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
+
+    - name: Semgrep errors
+      run: |
         files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files
+        semgrep --config="r/python.lang.correctness" --quiet --error $files
+
+    - name: Semgrep warnings
+      run: |
+        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files

README.md

@@ -39,7 +39,8 @@ Full-stack web application framework that uses Python and MariaDB on the server
 
 ### Installation
 
-[Install via Frappe Bench](https://github.com/frappe/bench)
+* [Install via Docker](https://github.com/frappe/frappe_docker)
+* [Install via Frappe Bench](https://github.com/frappe/bench)
 
 ## Contributing
 

cypress/integration/relative_time_filters.js

@@ -1,7 +1,4 @@
 context('Relative Timeframe', () => {
-	beforeEach(() => {
-		cy.login();
-	});
 	before(() => {
 		cy.login();
 		cy.visit('/app/website');

cypress/integration/table_multiselect.js

@@ -1,5 +1,5 @@
 context('Table MultiSelect', () => {
-	beforeEach(() => {
+	before(() => {
 		cy.login();
 	});
 

frappe/__init__.py

@@ -34,7 +34,7 @@ if PY2:
 	reload(sys)
 	sys.setdefaultencoding("utf-8")
 
-__version__ = '13.0.0-dev'
+__version__ = '13.1.0'
 
 __title__ = "Frappe Framework"
 
@@ -975,7 +975,7 @@ def get_pymodule_path(modulename, *joins):
 	:param *joins: Join additional path elements using `os.path.join`."""
 	if not "public" in joins:
 		joins = [scrub(part) for part in joins]
-	return os.path.join(os.path.dirname(get_module(scrub(modulename)).__file__), *joins)
+	return os.path.join(os.path.dirname(get_module(scrub(modulename)).__file__ or ''), *joins)
 
 def get_module_list(app_name):
 	"""Get list of modules for given all via `app/modules.txt`."""

frappe/api.py

@@ -1,12 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-from __future__ import unicode_literals
 
 import base64
 import binascii
 import json
-
-from six.moves.urllib.parse import urlencode, urlparse
+from urllib.parse import urlencode, urlparse
 
 import frappe
 import frappe.client
@@ -14,6 +12,7 @@ import frappe.handler
 from frappe import _
 from frappe.utils.response import build_response
 
+
 def handle():
 	"""
 	Handler for `/api` methods
@@ -38,9 +37,6 @@ def handle():
 	`/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method
 	"""
 
-
-	validate_auth()
-
 	parts = frappe.request.path[1:].split("/",3)
 	call = doctype = name = None
 
@@ -116,7 +112,7 @@ def handle():
 						frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields'])
 					frappe.local.form_dict.setdefault('limit_page_length', 20)
 					frappe.local.response.update({
-						"data":  frappe.call(
+						"data": frappe.call(
 							frappe.client.get_list,
 							doctype,
 							**frappe.local.form_dict
@@ -140,6 +136,7 @@ def handle():
 
 	return build_response("json")
 
+
 def get_request_form_data():
 	if frappe.local.form_dict.data is None:
 		data = frappe.safe_decode(frappe.local.request.get_data())
@@ -148,25 +145,18 @@ def get_request_form_data():
 
 	return frappe.parse_json(data)
 
+
 def validate_auth():
-	if frappe.get_request_header("Authorization") is None:
-		return
-
-	VALID_AUTH_PREFIX_TYPES = ['basic', 'bearer', 'token']
-	VALID_AUTH_PREFIX_STRING = ", ".join(VALID_AUTH_PREFIX_TYPES).title()
-
+	"""
+	Authenticate and sets user for the request.
+	"""
 	authorization_header = frappe.get_request_header("Authorization", str()).split(" ")
-	authorization_type = authorization_header[0].lower()
 
-	if len(authorization_header) == 1:
-		frappe.throw(_('Invalid Authorization headers, add a token with a prefix from one of the following: {0}.').format(VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationHeader)
-
-	if authorization_type == "bearer":
+	if len(authorization_header) == 2:
 		validate_oauth(authorization_header)
-	elif authorization_type in VALID_AUTH_PREFIX_TYPES:
 		validate_auth_via_api_keys(authorization_header)
-	else:
-		frappe.throw(_('Invalid Authorization Type {0}, must be one of {1}.').format(authorization_type, VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationPrefix)
+
+	validate_auth_via_hooks()
 
 
 def validate_oauth(authorization_header):
@@ -177,8 +167,8 @@ def validate_oauth(authorization_header):
 		authorization_header (list of str): The 'Authorization' header containing the prefix and token
 	"""
 
-	from frappe.oauth import get_url_delimiter
 	from frappe.integrations.oauth2 import get_oauth_server
+	from frappe.oauth import get_url_delimiter
 
 	form_dict = frappe.local.form_dict
 	token = authorization_header[1]
@@ -187,19 +177,20 @@ def validate_oauth(authorization_header):
 	access_token = {"access_token": token}
 	uri = parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path + "?" + urlencode(access_token)
 	http_method = req.method
-	body = req.get_data()
 	headers = req.headers
+	body = req.get_data()
+	if req.content_type and "multipart/form-data" in req.content_type:
+		body = None
 
 	try:
 		required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter())
+		valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes)
+		if valid:
+			frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
+			frappe.local.form_dict = form_dict
 	except AttributeError:
-		frappe.throw(_("Invalid Bearer token, please provide a valid access token with prefix 'Bearer'."), frappe.InvalidAuthorizationToken)
+		pass
 
-	valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes)
-
-	if valid:
-		frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
-		frappe.local.form_dict = form_dict
 
 
 def validate_auth_via_api_keys(authorization_header):
@@ -222,8 +213,7 @@ def validate_auth_via_api_keys(authorization_header):
 	except binascii.Error:
 		frappe.throw(_("Failed to decode token, please provide a valid base64-encoded token."), frappe.InvalidAuthorizationToken)
 	except (AttributeError, TypeError, ValueError):
-		frappe.throw(_("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'."), frappe.InvalidAuthorizationToken)
-
+		pass
 
 
 def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None):
@@ -248,3 +238,8 @@ def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=Non
 		if frappe.local.login_manager.user in ('', 'Guest'):
 			frappe.set_user(user)
 		frappe.local.form_dict = form_dict
+
+
+def validate_auth_via_hooks():
+	for auth_hook in frappe.get_hooks('auth_hooks', []):
+		frappe.get_attr(auth_hook)()

frappe/app.py

@@ -55,6 +55,7 @@ def application(request):
 		frappe.recorder.record()
 		frappe.monitor.start()
 		frappe.rate_limiter.apply()
+		frappe.api.validate_auth()
 
 		if request.method == "OPTIONS":
 			response = Response()

frappe/auth.py

@@ -120,6 +120,7 @@ class LoginManager:
 				self.make_session()
 				self.set_user_info()
 
+	@frappe.whitelist()
 	def login(self):
 		# clear cache
 		frappe.clear_cache(user = frappe.form_dict.get('usr'))

frappe/cache_manager.py

@@ -18,7 +18,7 @@ global_cache_keys = ("app_hooks", "installed_apps", 'all_apps',
 		'scheduler_events', 'time_zone', 'webhooks', 'active_domains',
 		'active_modules', 'assignment_rule', 'server_script_map', 'wkhtmltopdf_version',
 		'domain_restricted_doctypes', 'domain_restricted_pages', 'information_schema:counts',
-		'sitemap_routes', 'db_tables') + doctype_map_keys
+		'sitemap_routes', 'db_tables', 'server_script_autocompletion_items') + doctype_map_keys
 
 user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang",
 		"defaults", "user_permissions", "home_page", "linked_with",

frappe/change_log/v13/v13_1_0.md

@@ -0,0 +1,22 @@
+# Version 13.1.0 Release Notes
+
+### Features & Enhancements
+
+- Automated mail notifications will be shown in timeline ([#12693](https://github.com/frappe/frappe/pull/12693))
+- Introduced Client Script for List views ([#12590](https://github.com/frappe/frappe/pull/12590))
+- Introduced language switcher for guest users on website navbar ([#12813](https://github.com/frappe/frappe/pull/12813))
+- Option to give submit permission while sharing a document ([#12799](https://github.com/frappe/frappe/pull/12799))
+- Added option to set `autoname` in Customize Form ([#12413](https://github.com/frappe/frappe/pull/12413))
+- Virtual DocType ([#12121](https://github.com/frappe/frappe/pull/12121))
+
+### Fixes
+
+- Workspace fixes ([#12650](https://github.com/frappe/frappe/pull/12650)) ([#12655](https://github.com/frappe/frappe/pull/12655)) ([#12869](https://github.com/frappe/frappe/pull/12869))
+- Fixed an issue where select options were not getting updated in Grid ([#12839](https://github.com/frappe/frappe/pull/12839))
+- Webform Fixes ([#12630](https://github.com/frappe/frappe/pull/12630)) ([#12756](https://github.com/frappe/frappe/pull/12756)) ([#12819](https://github.com/frappe/frappe/pull/12819))
+- Fixed timespan filter for next and last timespans ([#12509](https://github.com/frappe/frappe/pull/12509))
+- System Notification fixes ([#12719](https://github.com/frappe/frappe/pull/12719))
+- Design Fixes ([#12669](https://github.com/frappe/frappe/pull/12669)) ([#12591](https://github.com/frappe/frappe/pull/12591)) ([#12557](https://github.com/frappe/frappe/pull/12557)) ([#12751](https://github.com/frappe/frappe/pull/12751)) ([#12864](https://github.com/frappe/frappe/pull/12864))
+- Fixed Multi-column paste in grid ([#12861](https://github.com/frappe/frappe/pull/12861))
+- Fixed grid validation ([#12744](https://github.com/frappe/frappe/pull/12744))
+- Fixed currency value formatting in dashboard chart ([#12613](https://github.com/frappe/frappe/pull/12613))

frappe/commands/__init__.py

@@ -62,11 +62,24 @@ def popen(command, *args, **kwargs):
 	if env:
 		env = dict(environ, **env)
 
+	def set_low_prio():
+		import psutil
+		if psutil.LINUX:
+			psutil.Process().nice(19)
+			psutil.Process().ionice(psutil.IOPRIO_CLASS_IDLE)
+		elif psutil.WINDOWS:
+			psutil.Process().nice(psutil.IDLE_PRIORITY_CLASS)
+			psutil.Process().ionice(psutil.IOPRIO_VERYLOW)
+		else:
+			psutil.Process().nice(19)
+			# ionice not supported
+
 	proc = subprocess.Popen(command,
 		stdout=None if output else subprocess.PIPE,
 		stderr=None if output else subprocess.PIPE,
 		shell=shell,
 		cwd=cwd,
+		preexec_fn=set_low_prio,
 		env=env
 	)
 

frappe/commands/scheduler.py

@@ -18,22 +18,33 @@ def _is_scheduler_enabled():
 
 	return enable_scheduler
 
-@click.command('trigger-scheduler-event')
-@click.argument('event')
+
+@click.command("trigger-scheduler-event", help="Trigger a scheduler event")
+@click.argument("event")
 @pass_context
 def trigger_scheduler_event(context, event):
-	"Trigger a scheduler event"
 	import frappe.utils.scheduler
+
+	exit_code = 0
+
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			frappe.utils.scheduler.trigger(site, event, now=True)
+			try:
+				frappe.get_doc("Scheduled Job Type", {"method": event}).execute()
+			except frappe.DoesNotExistError:
+				click.secho(f"Event {event} does not exist!", fg="red")
+				exit_code = 1
 		finally:
 			frappe.destroy()
+
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
+	sys.exit(exit_code)
+
+
 @click.command('enable-scheduler')
 @pass_context
 def enable_scheduler(context):

frappe/commands/site.py

@@ -676,10 +676,8 @@ def start_ngrok(context):
 	frappe.init(site=site)
 
 	port = frappe.conf.http_port or frappe.conf.webserver_port
-	public_url = ngrok.connect(port=port, options={
-		'host_header': site
-	})
-	print(f'Public URL: {public_url}')
+	tunnel = ngrok.connect(addr=str(port), host_header=site)
+	print(f'Public URL: {tunnel.public_url}')
 	print('Inspect logs at http://localhost:4040')
 
 	ngrok_process = ngrok.get_ngrok_process()

frappe/commands/utils.py

@@ -11,7 +11,7 @@ import click
 import frappe
 from frappe.commands import get_site, pass_context
 from frappe.exceptions import SiteNotSpecifiedError
-from frappe.utils import get_bench_path, update_progress_bar
+from frappe.utils import get_bench_path, update_progress_bar, cint
 
 
 @click.command('build')
@@ -567,11 +567,14 @@ def run_ui_tests(context, app, headless=False):
 
 	node_bin = subprocess.getoutput("npm bin")
 	cypress_path = "{0}/cypress".format(node_bin)
-	plugin_path = "{0}/cypress-file-upload".format(node_bin)
+	plugin_path = "{0}/../cypress-file-upload".format(node_bin)
 
 	# check if cypress in path...if not, install it.
-	if not (os.path.exists(cypress_path) or os.path.exists(plugin_path)) \
-		or not subprocess.getoutput("npm view cypress version").startswith("6."):
+	if not (
+		os.path.exists(cypress_path)
+		and os.path.exists(plugin_path)
+		and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6
+	):
 		# install cypress
 		click.secho("Installing Cypress...", fg="yellow")
 		frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 --no-lockfile")

frappe/core/doctype/report/report.py

@@ -325,9 +325,8 @@ def get_group_by_field(args, doctype):
 	if args['aggregate_function'] == 'count':
 		group_by_field = 'count(*) as _aggregate_column'
 	else:
-		group_by_field = '{0}(`tab{1}`.{2}) as _aggregate_column'.format(
+		group_by_field = '{0}({1}) as _aggregate_column'.format(
 			args.aggregate_function,
-			doctype,
 			args.aggregate_on
 		)
 

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -2,14 +2,15 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
+import json
+from datetime import datetime
 from typing import Dict, List
 
-import frappe, json
-from frappe.model.document import Document
-from frappe.utils import now_datetime, get_datetime
-from datetime import datetime
 from croniter import croniter
+
+import frappe
+from frappe.model.document import Document
+from frappe.utils import get_datetime, now_datetime
 from frappe.utils.background_jobs import enqueue, get_jobs
 
 

frappe/core/doctype/server_script/server_script.js

@@ -9,6 +9,12 @@ frappe.ui.form.on('Server Script', {
 		if (frm.doc.script_type != 'Scheduler Event') {
 			frm.dashboard.hide();
 		}
+
+		frm.call('get_autocompletion_items')
+			.then(r => r.message)
+			.then(items => {
+				frm.set_df_property('script', 'autocompletions', items);
+			});
 	},
 
 	setup_help(frm) {

frappe/core/doctype/server_script/server_script.py

@@ -5,11 +5,12 @@
 from __future__ import unicode_literals
 
 import ast
+from types import FunctionType, MethodType, ModuleType
 from typing import Dict, List
 
 import frappe
 from frappe.model.document import Document
-from frappe.utils.safe_exec import safe_exec
+from frappe.utils.safe_exec import get_safe_globals, safe_exec, NamespaceDict
 from frappe import _
 
 
@@ -122,6 +123,51 @@ class ServerScript(Document):
 		if locals["conditions"]:
 			return locals["conditions"]
 
+	@frappe.whitelist()
+	def get_autocompletion_items(self):
+		"""Generates a list of a autocompletion strings from the context dict
+		that is used while executing a Server Script.
+
+		Returns:
+			list: Returns list of autocompletion items.
+			For e.g., ["frappe.utils.cint", "frappe.db.get_all", ...]
+		"""
+		def get_keys(obj):
+			out = []
+			for key in obj:
+				if key.startswith('_'):
+					continue
+				value = obj[key]
+				if isinstance(value, (NamespaceDict, dict)) and value:
+					if key == 'form_dict':
+						out.append(['form_dict', 7])
+						continue
+					for subkey, score in get_keys(value):
+						fullkey = f'{key}.{subkey}'
+						out.append([fullkey, score])
+				else:
+					if isinstance(value, type) and issubclass(value, Exception):
+						score = 0
+					elif isinstance(value, ModuleType):
+						score = 10
+					elif isinstance(value, (FunctionType, MethodType)):
+						score = 9
+					elif isinstance(value, type):
+						score = 8
+					elif isinstance(value, dict):
+						score = 7
+					else:
+						score = 6
+					out.append([key, score])
+			return out
+
+		items = frappe.cache().get_value('server_script_autocompletion_items')
+		if not items:
+			items = get_keys(get_safe_globals())
+			items = [{'value': d[0], 'score': d[1]} for d in items]
+			frappe.cache().set_value('server_script_autocompletion_items', items)
+		return items
+
 
 @frappe.whitelist()
 def setup_scheduler_events(script_name, frequency):

frappe/database/mariadb/database.py

@@ -1,17 +1,13 @@
-from __future__ import unicode_literals
-
-import frappe
 import warnings
 
 import pymysql
-from pymysql.times import TimeDelta
-from pymysql.constants 	import ER, FIELD_TYPE
-from pymysql.converters import conversions
+from pymysql.constants import ER, FIELD_TYPE
+from pymysql.converters import conversions, escape_string
 
-from frappe.utils import get_datetime, cstr, UnicodeWithAttrs
+import frappe
 from frappe.database.database import Database
-from six import PY2, binary_type, text_type, string_types
 from frappe.database.mariadb.schema import MariaDBTable
+from frappe.utils import UnicodeWithAttrs, cstr, get_datetime
 
 
 class MariaDBDatabase(Database):
@@ -72,22 +68,20 @@ class MariaDBDatabase(Database):
 		conversions.update({
 			FIELD_TYPE.NEWDECIMAL: float,
 			FIELD_TYPE.DATETIME: get_datetime,
-			UnicodeWithAttrs: conversions[text_type]
+			UnicodeWithAttrs: conversions[str]
 		})
 
-		if PY2:
-			conversions.update({
-				TimeDelta: conversions[binary_type]
-			})
-
-		if usessl:
-			conn = pymysql.connect(self.host, self.user or '', self.password or '',
-				port=self.port, charset='utf8mb4', use_unicode = True, ssl=ssl_params,
-				conv = conversions, local_infile = frappe.conf.local_infile)
-		else:
-			conn = pymysql.connect(self.host, self.user or '', self.password or '',
-				port=self.port, charset='utf8mb4', use_unicode = True, conv = conversions,
-				local_infile = frappe.conf.local_infile)
+		conn = pymysql.connect(
+			user=self.user or '',
+			password=self.password or '',
+			host=self.host,
+			port=self.port,
+			charset='utf8mb4',
+			use_unicode=True,
+			ssl=ssl_params if usessl else None,
+			conv=conversions,
+			local_infile=frappe.conf.local_infile
+		)
 
 		# MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1
 		# # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF)
@@ -111,7 +105,7 @@ class MariaDBDatabase(Database):
 	def escape(s, percent=True):
 		"""Excape quotes and percent in given string."""
 		# pymysql expects unicode argument to escape_string with Python 3
-		s = frappe.as_unicode(pymysql.escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`")
+		s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`")
 
 		# NOTE separating % escape, because % escape should only be done when using LIKE operator
 		# or when you use python format string to generate query that already has a %s
@@ -260,7 +254,7 @@ class MariaDBDatabase(Database):
 				ADD INDEX `%s`(%s)""" % (table_name, index_name, ", ".join(fields)))
 
 	def add_unique(self, doctype, fields, constraint_name=None):
-		if isinstance(fields, string_types):
+		if isinstance(fields, str):
 			fields = [fields]
 		if not constraint_name:
 			constraint_name = "unique_" + "_".join(fields)

frappe/desk/reportview.py

@@ -126,13 +126,14 @@ def setup_group_by(data):
 	if data.group_by:
 		if data.aggregate_function.lower() not in ('count', 'sum', 'avg'):
 			frappe.throw(_('Invalid aggregate function'))
-		if '`' in data.aggregate_on:
-			raise_invalid_field(data.aggregate_on)
-		data.fields.append('{aggregate_function}(`tab{doctype}`.`{aggregate_on}`) AS _aggregate_column'.format(**data))
-		if data.aggregate_on:
-			data.fields.append(data.aggregate_on)
 
-		data.pop('aggregate_on')
+		if frappe.db.has_column(data.aggregate_on_doctype, data.aggregate_on_field):
+			data.fields.append('{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column'.format(**data))
+		else:
+			raise_invalid_field(data.aggregate_on_field)
+
+		data.pop('aggregate_on_doctype')
+		data.pop('aggregate_on_field')
 		data.pop('aggregate_function')
 
 def raise_invalid_field(fieldname):

frappe/desk/treeview.py

@@ -36,7 +36,7 @@ def get_all_nodes(doctype, label, parent, tree_method, **filters):
 	return out
 
 @frappe.whitelist()
-def get_children(doctype, parent=''):
+def get_children(doctype, parent='', **filters):
 	return _get_children(doctype, parent)
 
 def _get_children(doctype, parent='', ignore_permissions=False):

frappe/email/doctype/newsletter/newsletter.py

@@ -24,6 +24,7 @@ class Newsletter(WebsiteGenerator):
 		if self.send_from:
 			validate_email_address(self.send_from, True)
 
+	@frappe.whitelist()
 	def test_send(self, doctype="Lead"):
 		self.recipients = frappe.utils.split_emails(self.test_email_id)
 		self.queue_all(test_email=True)

frappe/email/receive.py

@@ -1,18 +1,27 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
-import six
-from six import iteritems, text_type
-from six.moves import range
-import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re
-from email_reply_parser import EmailReplyParser
+import datetime
+import email
+import email.utils
+import imaplib
+import poplib
+import re
+import time
 from email.header import decode_header
+
+import _socket
+import chardet
+import six
+from email_reply_parser import EmailReplyParser
+
 import frappe
 from frappe import _, safe_decode, safe_encode
-from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now,
-	cint, cstr, strip, markdown, parse_addr)
-from frappe.core.doctype.file.file import get_random_filename, MaxFileSizeReachedError
+from frappe.core.doctype.file.file import (MaxFileSizeReachedError,
+	get_random_filename)
+from frappe.utils import (cint, convert_utc_to_user_timezone, cstr,
+	extract_email_id, markdown, now, parse_addr, strip)
+
 
 class EmailSizeExceededError(frappe.ValidationError): pass
 class EmailTimeoutError(frappe.ValidationError): pass
@@ -337,7 +346,7 @@ class EmailServer:
 			return
 
 		self.imap.select("Inbox")
-		for uid, operation in iteritems(uid_list):
+		for uid, operation in uid_list.items():
 			if not uid: continue
 
 			op = "+FLAGS" if operation == "Read" else "-FLAGS"
@@ -473,7 +482,7 @@ class Email:
 			self.html_content += markdown(text_content)
 
 	def get_charset(self, part):
-		"""Detect chartset."""
+		"""Detect charset."""
 		charset = part.get_content_charset()
 		if not charset:
 			charset = chardet.detect(safe_encode(cstr(part)))['encoding']
@@ -484,7 +493,7 @@ class Email:
 		charset = self.get_charset(part)
 
 		try:
-			return text_type(part.get_payload(decode=True), str(charset), "ignore")
+			return str(part.get_payload(decode=True), str(charset), "ignore")
 		except LookupError:
 			return part.get_payload()
 

frappe/handler.py

@@ -9,7 +9,6 @@ import frappe
 import frappe.utils
 import frappe.sessions
 from frappe.utils import cint
-from frappe.api import validate_auth
 from frappe import _, is_whitelisted
 from frappe.utils.response import build_response
 from frappe.utils.csvutils import build_csv_response
@@ -24,7 +23,7 @@ ALLOWED_MIMETYPES = ('image/png', 'image/jpeg', 'application/pdf', 'application/
 
 def handle():
 	"""handle request"""
-	validate_auth()
+
 	cmd = frappe.local.form_dict.cmd
 	data = None
 

frappe/hooks.py

@@ -130,6 +130,16 @@ has_website_permission = {
 	"Address": "frappe.contacts.doctype.address.address.has_website_permission"
 }
 
+jinja = {
+	"methods": "frappe.utils.jinja_globals",
+	"filters": [
+		"frappe.utils.data.global_date_format",
+		"frappe.utils.markdown",
+		"frappe.website.utils.get_shade",
+		"frappe.website.utils.abs_url",
+	]
+}
+
 standard_queries = {
 	"User": "frappe.core.doctype.user.user.user_query"
 }

frappe/integrations/doctype/dropbox_settings/dropbox_settings.py

@@ -2,22 +2,23 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import dropbox
 import json
-import frappe
 import os
-from frappe import _
-from frappe.model.document import Document
-from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site
-from frappe.integrations.utils import make_post_request
-from frappe.utils import (cint, get_request_site_address,
-	get_files_path, get_backups_path, get_url, encode)
-from frappe.utils.backups import new_backup
-from frappe.utils.background_jobs import enqueue
-from six.moves.urllib.parse import urlparse, parse_qs
+from urllib.parse import parse_qs, urlparse
+
+import dropbox
 from rq.timeouts import JobTimeoutException
-from six import text_type
+
+import frappe
+from frappe import _
+from frappe.integrations.offsite_backup_utils import (get_chunk_site,
+	get_latest_backup_file, send_email, validate_file_size)
+from frappe.integrations.utils import make_post_request
+from frappe.model.document import Document
+from frappe.utils import (cint, encode, get_backups_path, get_files_path,
+	get_request_site_address, get_url)
+from frappe.utils.background_jobs import enqueue
+from frappe.utils.backups import new_backup
 
 ignore_list = [".DS_Store"]
 
@@ -91,7 +92,10 @@ def backup_to_dropbox(upload_db_backup=True):
 		dropbox_settings['access_token'] = access_token['oauth2_token']
 		set_dropbox_access_token(access_token['oauth2_token'])
 
-	dropbox_client = dropbox.Dropbox(dropbox_settings['access_token'], timeout=None)
+	dropbox_client = dropbox.Dropbox(
+		oauth2_access_token=dropbox_settings['access_token'],
+		timeout=None
+	)
 
 	if upload_db_backup:
 		if frappe.flags.create_new_backup:
@@ -127,7 +131,7 @@ def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not
 	else:
 		response = frappe._dict({"entries": []})
 
-	path = text_type(path)
+	path = str(path)
 
 	for f in frappe.get_all("File", filters={"is_folder": 0, "is_private": is_private,
 		"uploaded_to_dropbox": 0}, fields=['file_url', 'name', 'file_name']):
@@ -286,11 +290,11 @@ def get_redirect_url():
 def get_dropbox_authorize_url():
 	app_details = get_dropbox_settings(redirect_uri=True)
 	dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
-		app_details["app_key"],
-		app_details["app_secret"],
-		app_details["redirect_uri"],
-		{},
-		"dropbox-auth-csrf-token"
+		consumer_key=app_details["app_key"],
+		redirect_uri=app_details["redirect_uri"],
+		session={},
+		csrf_token_session_key="dropbox-auth-csrf-token",
+		consumer_secret=app_details["app_secret"]
 	)
 
 	auth_url = dropbox_oauth_flow.start()
@@ -307,13 +311,13 @@ def dropbox_auth_finish(return_access_token=False):
 	close = '<p class="text-muted">' + _('Please close this window') + '</p>'
 
 	dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
-		app_details["app_key"],
-		app_details["app_secret"],
-		app_details["redirect_uri"],
-		{
+		consumer_key=app_details["app_key"],
+		redirect_uri=app_details["redirect_uri"],
+		session={
 			'dropbox-auth-csrf-token': callback.state
 		},
-		"dropbox-auth-csrf-token"
+		csrf_token_session_key="dropbox-auth-csrf-token",
+		consumer_secret=app_details["app_secret"]
 	)
 
 	if callback.state or callback.code:

frappe/integrations/doctype/google_calendar/google_calendar.py

@@ -2,22 +2,23 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
-import google.oauth2.credentials
 
-from frappe import _
-from frappe.model.document import Document
-from frappe.utils import get_request_site_address
-from googleapiclient.errors import HttpError
-from frappe.utils.password import set_encrypted_password
-from frappe.utils import add_days, get_datetime, get_weekdays, now_datetime, add_to_date, get_time_zone
-from dateutil import parser
 from datetime import datetime, timedelta
-from six.moves.urllib.parse import quote
+from urllib.parse import quote
+
+import google.oauth2.credentials
+import requests
+from dateutil import parser
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+import frappe
+from frappe import _
 from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.model.document import Document
+from frappe.utils import (add_days, add_to_date, get_datetime,
+	get_request_site_address, get_time_zone, get_weekdays, now_datetime)
+from frappe.utils.password import set_encrypted_password
 
 SCOPES = "https://www.googleapis.com/auth/calendar"
 
@@ -171,7 +172,12 @@ def get_google_calendar_object(g_calendar):
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_calendar = googleapiclient.discovery.build("calendar", "v3", credentials=credentials)
+	google_calendar = build(
+		serviceName="calendar",
+		version="v3",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	check_google_calendar(account, google_calendar)
 

frappe/integrations/doctype/google_contacts/google_contacts.py

@@ -2,17 +2,17 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
-import google.oauth2.credentials
 
-from frappe.model.document import Document
-from frappe import _
+import google.oauth2.credentials
+import requests
+from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
-from frappe.utils import get_request_site_address
+
+import frappe
+from frappe import _
 from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.model.document import Document
+from frappe.utils import get_request_site_address
 
 SCOPES = "https://www.googleapis.com/auth/contacts"
 
@@ -118,7 +118,12 @@ def get_google_contacts_object(g_contact):
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_contacts = googleapiclient.discovery.build("people", "v1", credentials=credentials)
+	google_contacts = build(
+		serviceName="people",
+		version="v1",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	return google_contacts, account
 

frappe/integrations/doctype/google_drive/google_drive.py

@@ -2,27 +2,29 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
-import google.oauth2.credentials
 import os
+from urllib.parse import quote
 
-from frappe import _
-from googleapiclient.errors import HttpError
-from frappe.model.document import Document
-from frappe.utils import get_request_site_address
-from frappe.utils.background_jobs import enqueue
-from six.moves.urllib.parse import quote
+import google.oauth2.credentials
+import requests
 from apiclient.http import MediaFileUpload
-from frappe.utils import get_backups_path, get_bench_path
-from frappe.utils.backups import new_backup
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+import frappe
+from frappe import _
 from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
-from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
+from frappe.integrations.offsite_backup_utils import (get_latest_backup_file,
+	send_email, validate_file_size)
+from frappe.model.document import Document
+from frappe.utils import (get_backups_path, get_bench_path,
+	get_request_site_address)
+from frappe.utils.background_jobs import enqueue
+from frappe.utils.backups import new_backup
 
 SCOPES = "https://www.googleapis.com/auth/drive"
 
+
 class GoogleDrive(Document):
 
 	def validate(self):
@@ -126,7 +128,12 @@ def get_google_drive_object():
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_drive = googleapiclient.discovery.build("drive", "v3", credentials=credentials)
+	google_drive = build(
+		serviceName="drive",
+		version="v3",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	return google_drive, account
 

frappe/integrations/doctype/token_cache/test_token_cache.py

@@ -13,7 +13,7 @@ class TestTokenCache(unittest.TestCase):
 	def setUp(self):
 		self.token_cache = frappe.get_last_doc('Token Cache')
 		self.token_cache.update({'connected_app': frappe.get_last_doc('Connected App').name})
-		self.token_cache.save()
+		self.token_cache.save(ignore_permissions=True)
 
 	def test_get_auth_header(self):
 		self.token_cache.get_auth_header()

frappe/integrations/oauth2.py

@@ -133,7 +133,7 @@ def get_token(*args, **kwargs):
 			}
 
 			id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header)
-			out.update({"id_token": str(id_token_encoded)})
+			out.update({"id_token": frappe.safe_decode(id_token_encoded)})
 
 		frappe.local.response = out
 

frappe/patches.txt

@@ -335,3 +335,4 @@ frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings
 frappe.patches.v13_0.remove_twilio_settings
 frappe.patches.v12_0.rename_uploaded_files_with_proper_name
 frappe.patches.v13_0.queryreport_columns
+frappe.patches.v13_0.jinja_hook

frappe/patches/v13_0/jinja_hook.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+from click import secho
+
+def execute():
+	if frappe.get_hooks('jenv'):
+		print()
+		secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow')
+		secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow')
+		print()

frappe/public/js/frappe/form/controls/base_control.js

@@ -159,9 +159,13 @@ frappe.ui.form.Control = Class.extend({
 	},
 	validate_and_set_in_model: function(value, e) {
 		var me = this;
-		if(this.inside_change_event) {
+		let force_value_set = (this.doc && this.doc.__run_link_triggers);
+		let is_value_same = (this.get_model_value() === value);
+
+		if (this.inside_change_event || (!force_value_set && is_value_same)) {
 			return Promise.resolve();
 		}
+
 		this.inside_change_event = true;
 		var set = function(value) {
 			me.inside_change_event = false;

frappe/public/js/frappe/form/controls/code.js

@@ -31,6 +31,57 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({
 			const input_value = this.get_input_value();
 			this.parse_validate_and_set_in_model(input_value);
 		}, 300));
+
+		// setup autocompletion when it is set the first time
+		Object.defineProperty(this.df, 'autocompletions', {
+			get() {
+				return this._autocompletions || [];
+			},
+			set: (value) => {
+				this.setup_autocompletion();
+				this.df._autocompletions = value;
+			}
+		});
+	},
+
+	setup_autocompletion() {
+		if (this._autocompletion_setup) return;
+
+		const ace = window.ace;
+		const get_autocompletions = () => this.df.autocompletions;
+
+		ace.config.loadModule("ace/ext/language_tools", langTools => {
+			this.editor.setOptions({
+				enableBasicAutocompletion: true,
+				enableLiveAutocompletion: true
+			});
+
+			langTools.addCompleter({
+				getCompletions: function(editor, session, pos, prefix, callback) {
+					if (prefix.length === 0) {
+						callback(null, []);
+						return;
+					}
+					let autocompletions = get_autocompletions();
+					if (autocompletions.length) {
+						callback(
+							null,
+							autocompletions.map(a => {
+								if (typeof a === 'string') {
+									a = { value: a };
+								}
+								return {
+									name: 'frappe',
+									value: a.value,
+									score: a.score
+								};
+							})
+						);
+					}
+				}
+			});
+		});
+		this._autocompletion_setup = true;
 	},
 
 	refresh_height() {

frappe/public/js/frappe/form/controls/link.js

@@ -462,9 +462,10 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({
 				if(this.frm && this.frm.fetch_dict[df.fieldname]) {
 					fetch = this.frm.fetch_dict[df.fieldname].columns.join(', ');
 				}
-
 				// if default and no fetch, no need to validate
-				if (!fetch && df.__default_value && df.__default_value===value) return value;
+				if (!fetch && df.__default_value && df.__default_value===value) {
+					resolve(value);
+				}
 
 				this.fetch_and_validate_link(resolve, df, doctype, docname, value, fetch);
 			});

frappe/public/js/frappe/form/controls/table_multiselect.js

@@ -66,6 +66,10 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({
 		this._rows_list = this.rows.map(row => row[link_field.fieldname]);
 		return this.rows;
 	},
+	get_model_value() {
+		let value = this._super();
+		return value ? value.filter(d => !d.__islocal) : value;
+	},
 	validate(value) {
 		const rows = (value || []).slice();
 

frappe/public/js/frappe/form/dashboard.js

@@ -290,7 +290,7 @@ frappe.ui.form.Dashboard = class FormDashboard {
 
 		// bind links
 		transactions_area_body.find(".badge-link").on('click', function() {
-			me.open_document_list($(this).parent());
+			me.open_document_list($(this).closest('.document-link'));
 		});
 
 		// bind reports

frappe/public/js/frappe/form/form.js

@@ -360,6 +360,7 @@ frappe.ui.form.Form = class FrappeForm {
 			grid_obj.grid.grid_pagination.go_to_page(1, true);
 		});
 		frappe.ui.form.close_grid_form();
+		this.viewers && this.viewers.parent.empty();
 		this.docname = docname;
 		this.setup_docinfo_change_listener();
 	}
@@ -1202,8 +1203,7 @@ frappe.ui.form.Form = class FrappeForm {
 
 		$.each(grid_field_label_map, function(fname, label) {
 			fname = fname.split("-");
-			var df = frappe.meta.get_docfield(fname[0], fname[1], me.doc.name);
-			if(df) df.label = label;
+			me.fields_dict[parentfield].grid.update_docfield_property(fname[1], 'label', label);
 		});
 	}
 

frappe/public/js/frappe/form/grid.js

@@ -387,6 +387,8 @@ export default class Grid {
 			this.wrapper.find('.grid-footer').toggle(false);
 		}
 
+		this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable());
+
 	}
 
 	truncate_rows() {

frappe/public/js/frappe/form/grid_row.js

@@ -6,6 +6,7 @@ export default class GridRow {
 		this.on_grid_fields = [];
 		$.extend(this, opts);
 		if (this.doc && this.parent_df.options) {
+			frappe.meta.make_docfield_copy_for(this.parent_df.options, this.doc.name, this.docfields);
 			this.docfields = frappe.meta.get_docfields(this.parent_df.options, this.doc.name);
 		}
 		this.columns = {};
@@ -555,6 +556,12 @@ export default class GridRow {
 		this.grid_form.render();
 		this.row.toggle(false);
 		// this.form_panel.toggle(true);
+
+		let cannot_add_rows = this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows);
+		this.wrapper
+			.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row')
+			.toggle(!cannot_add_rows);
+
 		frappe.dom.freeze("", "dark");
 		if (cur_frm) cur_frm.cur_grid = this;
 		this.wrapper.addClass("grid-row-open");

frappe/public/js/frappe/form/grid_row_form.js

@@ -119,7 +119,7 @@ export default class GridRowForm {
 		});
 	}
 	toggle_add_delete_button_display($parent) {
-		$parent.find(".row-actions")
+		$parent.find(".row-actions, .grid-append-row")
 			.toggle(this.row.grid.is_editable());
 	}
 	refresh_field(fieldname) {

frappe/public/js/frappe/form/toolbar.js

@@ -211,7 +211,6 @@ frappe.ui.form.Toolbar = class Toolbar {
 
 	make_viewers() {
 		if (this.frm.viewers) {
-			this.frm.viewers.parent.empty();
 			return;
 		}
 		this.frm.viewers = new frappe.ui.form.FormViewers({

frappe/public/js/frappe/model/create_new.js

@@ -177,7 +177,9 @@ $.extend(frappe.model, {
 						// Use User Permission value when only when it has a single value
 						user_default = user_defaults[0];
 					}
-				} else if (!user_default) {
+				}
+				
+				if (!user_default) {
 					user_default = frappe.defaults.get_user_default(df.fieldname);
 				} else if (
 					!user_default &&

frappe/public/js/frappe/model/meta.js

@@ -38,14 +38,14 @@ $.extend(frappe.meta, {
 		frappe.meta.docfield_list[df.parent].push(df);
 	},
 
-	make_docfield_copy_for: function(doctype, docname) {
+	make_docfield_copy_for: function(doctype, docname, docfield_list=null) {
 		var c = frappe.meta.docfield_copy;
 		if(!c[doctype])
 			c[doctype] = {};
 		if(!c[doctype][docname])
 			c[doctype][docname] = {};
 
-		var docfield_list = frappe.meta.docfield_list[doctype] || [];
+		docfield_list = docfield_list || frappe.meta.docfield_list[doctype] || [];
 		for(var i=0, j=docfield_list.length; i<j; i++) {
 			var df = docfield_list[i];
 			c[doctype][docname][df.fieldname || df.label] = copy_dict(df);

frappe/public/js/frappe/ui/group_by/group_by.js

@@ -313,7 +313,8 @@ frappe.ui.GroupBy = class {
 
 			Object.assign(args, {
 				with_comment_count: false,
-				aggregate_on: this.aggregate_on || 'name',
+				aggregate_on_field: this.aggregate_on_field || 'name',
+				aggregate_on_doctype: this.aggregate_on_doctype || this.doctype,
 				aggregate_function: this.aggregate_function || 'count',
 				group_by: this.report_view.group_by || null,
 				order_by: '_aggregate_column desc',

frappe/public/js/frappe/widgets/chart_widget.js

@@ -25,7 +25,6 @@ export default class ChartWidget extends Widget {
 		delete this.dashboard_chart;
 		this.set_body();
 		this.make_chart();
-		this.setup_events();
 	}
 
 	set_chart_title() {
@@ -747,18 +746,4 @@ export default class ChartWidget extends Widget {
 				}
 			});
 	}
-
-	setup_events() {
-		$(document.body).on('toggleSidebar', () => {
-			this.dashboard_chart && this.dashboard_chart.draw(true);
-		});
-
-		$(document.body).on('toggleListSidebar', () => {
-			this.dashboard_chart && this.dashboard_chart.draw(true);
-		});
-
-		$(document.body).on('toggleFullWidth', () => {
-			this.dashboard_chart && this.dashboard_chart.draw(true);
-		});
-	}
 }

frappe/templates/base.html

@@ -60,7 +60,7 @@
 		window.is_chat_enabled = {{ chat_enable }};
 	</script>
 </head>
-<body frappe-session-status="{{ 'logged-in' if frappe.session.user != 'Guest' else 'logged-out'}}" data-path="{{ path | e }}" {%- if template and template.endswith('.md') %} frappe-content-type="markdown" {% endif -%} class="{{ body_class or ''}}">
+<body frappe-session-status="{{ 'logged-in' if frappe.session.user != 'Guest' else 'logged-out'}}" data-path="{{ path | e }}" {%- if template and template.endswith('.md') %} frappe-content-type="markdown" {%- endif %} class="{{ body_class or ''}}">
 	{% include "public/icons/timeless/symbol-defs.svg" %}
 	{%- block banner -%}
 		{% include "templates/includes/banner_extension.html" ignore missing %}

frappe/translate.py

@@ -115,7 +115,7 @@ def get_dict(fortype, name=None):
 				messages.extend(get_server_messages(app))
 			messages = deduplicate_messages(messages)
 
-			messages += frappe.db.sql("""select "navbar", item_label from `tabNavbar Item` where item_label is not null""")
+			messages += frappe.db.sql("""select 'navbar', item_label from `tabNavbar Item` where item_label is not null""")
 			messages = get_messages_from_include_files()
 			messages += frappe.db.sql("select 'Print Format:', name from `tabPrint Format`")
 			messages += frappe.db.sql("select 'DocType:', name from tabDocType")
@@ -518,8 +518,13 @@ def get_messages_from_file(path):
 	apps_path = get_bench_dir()
 	if os.path.exists(path):
 		with open(path, 'r') as sourcefile:
+			try:
+				file_contents = sourcefile.read()
+			except Exception:
+				print("Could not scan file for translation: {0}".format(path))
+				return []
 			data = [(os.path.relpath(path, apps_path), message, context, line) \
-				for line, message, context in  extract_messages_from_code(sourcefile.read())]
+				for line, message, context in extract_messages_from_code(file_contents)]
 			return data
 	else:
 		# print "Translate: {0} missing".format(os.path.abspath(path))
@@ -601,11 +606,23 @@ def write_csv_file(path, app_messages, lang_dict):
 	from csv import writer
 	with open(path, 'w', newline='') as msgfile:
 		w = writer(msgfile, lineterminator='\n')
-		for p, m in app_messages:
-			t = lang_dict.get(m, '')
+
+		for app_message in app_messages:
+			context = None
+			if len(app_message) == 2:
+				path, message = app_message
+			elif len(app_message) == 3:
+				path, message, lineno = app_message
+			elif len(app_message) == 4:
+				path, message, context, lineno = app_message
+			else:
+				continue
+
+			t = lang_dict.get(message, '')
 			# strip whitespaces
-			t = re.sub('{\s?([0-9]+)\s?}', "{\g<1>}", t)
-			w.writerow([p if p else '', m, t])
+			translated_string = re.sub('{\s?([0-9]+)\s?}', "{\g<1>}", t)
+			if translated_string:
+				w.writerow([message, translated_string, context])
 
 def get_untranslated(lang, untranslated_file, get_all=False):
 	"""Returns all untranslated strings for a language and writes in a file
@@ -827,7 +844,7 @@ def get_all_languages(with_language_name=False):
 		return frappe.db.sql_list('select name from tabLanguage')
 
 	def get_all_language_with_name():
-		return frappe.db.get_all('language', ['language_code', 'language_name'])
+		return frappe.db.get_all('Language', ['language_code', 'language_name'])
 
 	if not frappe.db:
 		frappe.connect()

frappe/utils/__init__.py

@@ -307,14 +307,23 @@ def unesc(s, esc_chars):
 		s = s.replace(esc_str, c)
 	return s
 
-def execute_in_shell(cmd, verbose=0):
+def execute_in_shell(cmd, verbose=0, low_priority=False):
 	# using Popen instead of os.system - as recommended by python docs
 	import tempfile
 	from subprocess import Popen
 
 	with tempfile.TemporaryFile() as stdout:
 		with tempfile.TemporaryFile() as stderr:
-			p = Popen(cmd, shell=True, stdout=stdout, stderr=stderr)
+			kwargs = {
+				"shell": True,
+				"stdout": stdout,
+				"stderr": stderr
+			}
+
+			if low_priority:
+				kwargs["preexec_fn"] = lambda: os.nice(10)
+
+			p = Popen(cmd, **kwargs)
 			p.wait()
 
 			stdout.seek(0)

frappe/utils/backups.py

@@ -15,7 +15,7 @@ import click
 # imports - module imports
 import frappe
 from frappe import _, conf
-from frappe.utils import get_file_size, get_url, now, now_datetime
+from frappe.utils import get_file_size, get_url, now, now_datetime, cint
 
 # backup variable for backwards compatibility
 verbose = False
@@ -315,8 +315,6 @@ class BackupGenerator:
 			print(template.format(_type.title(), info["path"], info["size"]))
 
 	def backup_files(self):
-		import subprocess
-
 		for folder in ("public", "private"):
 			files_path = frappe.get_site_path(folder, "files")
 			backup_path = (
@@ -327,12 +325,12 @@ class BackupGenerator:
 				cmd_string = "tar cf - {1} | gzip > {0}"
 			else:
 				cmd_string = "tar -cf {0} {1}"
-			output = subprocess.check_output(
-				cmd_string.format(backup_path, files_path), shell=True
-			)
 
-			if self.verbose and output:
-				print(output.decode("utf8"))
+			frappe.utils.execute_in_shell(
+				cmd_string.format(backup_path, files_path),
+				verbose=self.verbose,
+				low_priority=True
+			)
 
 	def copy_site_config(self):
 		site_config_backup_path = self.backup_path_conf
@@ -436,7 +434,7 @@ class BackupGenerator:
 		if self.verbose:
 			print(command + "\n")
 
-		err, out = frappe.utils.execute_in_shell(command)
+		frappe.utils.execute_in_shell(command, low_priority=True)
 
 	def send_email(self):
 		"""
@@ -474,29 +472,6 @@ download only after 24 hours.""" % {
 		return recipient_list
 
 
-@frappe.whitelist()
-def get_backup():
-	"""
-	This function is executed when the user clicks on
-	Toos > Download Backup
-	"""
-	delete_temp_backups()
-	odb = BackupGenerator(
-		frappe.conf.db_name,
-		frappe.conf.db_name,
-		frappe.conf.db_password,
-		db_host=frappe.db.host,
-		db_type=frappe.conf.db_type,
-		db_port=frappe.conf.db_port,
-	)
-	odb.get_backup()
-	recipient_list = odb.send_email()
-	frappe.msgprint(
-		_(
-			"Download link for your backup will be emailed on the following email address: {0}"
-		).format(", ".join(recipient_list))
-	)
-
 @frappe.whitelist()
 def fetch_latest_backups(partial=False):
 	"""Fetches paths of the latest backup taken in the last 30 days
@@ -570,7 +545,7 @@ def new_backup(
 	force=False,
 	verbose=False,
 ):
-	delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24)
+	delete_temp_backups()
 	odb = BackupGenerator(
 		frappe.conf.db_name,
 		frappe.conf.db_name,
@@ -595,8 +570,9 @@ def new_backup(
 
 def delete_temp_backups(older_than=24):
 	"""
-	Cleans up the backup_link_path directory by deleting files older than 24 hours
+	Cleans up the backup_link_path directory by deleting older files
 	"""
+	older_than = cint(frappe.conf.keep_backups_for_hours) or older_than
 	backup_path = get_backup_path()
 	if os.path.exists(backup_path):
 		file_list = os.listdir(get_backup_path())

frappe/utils/boilerplate.py

@@ -190,6 +190,15 @@ app_license = "{app_license}"
 # automatically create page for each record of this doctype
 # website_generators = ["Web Page"]
 
+# Jinja
+# ----------
+
+# add methods and filters to jinja environment
+# jinja = {{
+# 	"methods": "{app_name}.utils.jinja_methods",
+# 	"filters": "{app_name}.utils.jinja_filters"
+# }}
+
 # Installation
 # ------------
 
@@ -303,6 +312,13 @@ user_data_fields = [
 	}}
 ]
 
+# Authentication and authorization
+# --------------------------------
+
+# auth_hooks = [
+# 	"{app_name}.auth.validate"
+# ]
+
 """
 
 desktop_template = """# -*- coding: utf-8 -*-

frappe/utils/jinja.py

@@ -18,13 +18,10 @@ def get_jenv():
 		set_filters(jenv)
 
 		jenv.globals.update(get_safe_globals())
-		jenv.globals.update(get_jenv_customization('methods'))
-		jenv.globals.update({
-			'resolve_class': resolve_class,
-			'inspect': inspect,
-			'web_blocks': web_blocks,
-			'web_block': web_block
-		})
+
+		methods, filters = get_jinja_hooks()
+		jenv.globals.update(methods or {})
+		jenv.filters.update(filters or {})
 
 		frappe.local.jenv = jenv
 
@@ -127,104 +124,49 @@ def get_jloader():
 
 def set_filters(jenv):
 	import frappe
-	from frappe.utils import global_date_format, cint, cstr, flt, markdown
-	from frappe.website.utils import get_shade, abs_url
+	from frappe.utils import cint, cstr, flt
 
-	jenv.filters["global_date_format"] = global_date_format
-	jenv.filters["markdown"] = markdown
 	jenv.filters["json"] = frappe.as_json
-	jenv.filters["get_shade"] = get_shade
 	jenv.filters["len"] = len
 	jenv.filters["int"] = cint
 	jenv.filters["str"] = cstr
 	jenv.filters["flt"] = flt
-	jenv.filters["abs_url"] = abs_url
 
 	if frappe.flags.in_setup_help:
 		return
 
-	jenv.filters.update(get_jenv_customization('filters'))
-
-
-def get_jenv_customization(customization_type):
-	'''Returns a dict with filter/method name as key and definition as value'''
 
+def get_jinja_hooks():
+	"""Returns a tuple of (methods, filters) each containing a dict of method name and method definition pair."""
 	import frappe
 
-	out = {}
 	if not getattr(frappe.local, "site", None):
+		return (None, None)
+
+	from types import FunctionType, ModuleType
+	from inspect import getmembers, isfunction
+
+	def get_obj_dict_from_paths(object_paths):
+		out = {}
+		for obj_path in object_paths:
+			try:
+				obj = frappe.get_module(obj_path)
+			except ModuleNotFoundError:
+				obj = frappe.get_attr(obj_path)
+
+			if isinstance(obj, ModuleType):
+				functions = getmembers(obj, isfunction)
+				for function_name, function in functions:
+					out[function_name] = function
+			elif isinstance(obj, FunctionType):
+				function_name = obj.__name__
+				out[function_name] = obj
 		return out
 
-	values = frappe.get_hooks("jenv", {}).get(customization_type)
-	if not values:
-		return out
+	values = frappe.get_hooks("jinja")
+	methods, filters = values.get("methods", []), values.get("filters", [])
 
-	for value in values:
-		fn_name, fn_string = value.split(":")
-		out[fn_name] = frappe.get_attr(fn_string)
+	method_dict = get_obj_dict_from_paths(methods)
+	filter_dict = get_obj_dict_from_paths(filters)
 
-	return out
-
-
-def resolve_class(classes):
-	import frappe
-
-	if classes is None:
-		return ''
-
-	if isinstance(classes, frappe.string_types):
-		return classes
-
-	if isinstance(classes, (list, tuple)):
-		return ' '.join([resolve_class(c) for c in classes]).strip()
-
-	if isinstance(classes, dict):
-		return ' '.join([classname for classname in classes if classes[classname]]).strip()
-
-	return classes
-
-
-def inspect(var, render=True):
-	context = { "var": var }
-	if render:
-		html = "<pre>{{ var | pprint | e }}</pre>"
-	else:
-		html = ""
-	return get_jenv().from_string(html).render(context)
-
-
-def web_block(template, values=None, **kwargs):
-	options = {"template": template, "values": values}
-	options.update(kwargs)
-	return web_blocks([options])
-
-
-def web_blocks(blocks):
-	from frappe import throw, _dict
-	from frappe.website.doctype.web_page.web_page import get_web_blocks_html
-
-	web_blocks = []
-	for block in blocks:
-		if not block.get('template'):
-			throw('Web Template is not specified')
-
-		doc = _dict({
-			'doctype': 'Web Page Block',
-			'web_template': block['template'],
-			'web_template_values': block.get('values', {}),
-			'add_top_padding': 1,
-			'add_bottom_padding': 1,
-			'add_container': 1,
-			'hide_block': 0,
-			'css_class': ''
-		})
-		doc.update(block)
-		web_blocks.append(doc)
-
-	out = get_web_blocks_html(web_blocks)
-
-	html = out.html
-	for script in out.scripts:
-		html += '<script>{}</script>'.format(script)
-
-	return html
+	return method_dict, filter_dict

frappe/utils/jinja_globals.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+from frappe.utils.jinja import get_jenv
+import frappe
+
+
+def resolve_class(classes):
+	if classes is None:
+		return ""
+
+	if isinstance(classes, frappe.string_types):
+		return classes
+
+	if isinstance(classes, (list, tuple)):
+		return " ".join([resolve_class(c) for c in classes]).strip()
+
+	if isinstance(classes, dict):
+		return " ".join([classname for classname in classes if classes[classname]]).strip()
+
+	return classes
+
+
+def inspect(var, render=True):
+	context = {"var": var}
+	if render:
+		html = "<pre>{{ var | pprint | e }}</pre>"
+	else:
+		return ""
+	return get_jenv().from_string(html).render(context)
+
+
+def web_block(template, values=None, **kwargs):
+	options = {"template": template, "values": values}
+	options.update(kwargs)
+	return web_blocks([options])
+
+
+def web_blocks(blocks):
+	from frappe import throw, _dict
+	from frappe.website.doctype.web_page.web_page import get_web_blocks_html
+
+	web_blocks = []
+	for block in blocks:
+		if not block.get("template"):
+			throw("Web Template is not specified")
+
+		doc = _dict(
+			{
+				"doctype": "Web Page Block",
+				"web_template": block["template"],
+				"web_template_values": block.get("values", {}),
+				"add_top_padding": 1,
+				"add_bottom_padding": 1,
+				"add_container": 1,
+				"hide_block": 0,
+				"css_class": "",
+			}
+		)
+		doc.update(block)
+		web_blocks.append(doc)
+
+	out = get_web_blocks_html(web_blocks)
+
+	html = out.html
+	for script in out.scripts:
+		html += "<script>{}</script>".format(script)
+
+	return html
+

frappe/utils/oauth.py

@@ -64,8 +64,6 @@ def get_oauth2_authorize_url(provider, redirect_to):
 
 	state = { "site": frappe.utils.get_url(), "token": frappe.generate_hash(), "redirect_to": redirect_to 	}
 
-	frappe.cache().set_value("{0}:{1}".format(provider, state["token"]), True, expires_in_sec=120)
-
 	# relative to absolute url
 	data = {
 		"redirect_uri": get_redirect_uri(provider),
@@ -176,11 +174,6 @@ def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=No
 		frappe.respond_as_web_page(_("Invalid Request"), _("Token is missing"), http_status_code=417)
 		return
 
-	token = frappe.cache().get_value("{0}:{1}".format(provider, state["token"]), expires=True)
-	if not token:
-		frappe.respond_as_web_page(_("Invalid Request"), _("Invalid Token"), http_status_code=417)
-		return
-
 	user = get_email(data)
 
 	if not user:

frappe/utils/xlsxutils.py

@@ -1,18 +1,19 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-from __future__ import unicode_literals
-
-import frappe
+import re
+from io import BytesIO
 
 import openpyxl
 import xlrd
-import re
-from openpyxl.styles import Font
 from openpyxl import load_workbook
+from openpyxl.styles import Font
 from openpyxl.utils import get_column_letter
-from six import BytesIO, string_types
+
+import frappe
 
 ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]')
+
+
 # return xlsx file object
 def make_xlsx(data, sheet_name, wb=None, column_widths=None):
 	column_widths = column_widths or []
@@ -31,12 +32,12 @@ def make_xlsx(data, sheet_name, wb=None, column_widths=None):
 	for row in data:
 		clean_row = []
 		for item in row:
-			if isinstance(item, string_types) and (sheet_name not in ['Data Import Template', 'Data Export']):
+			if isinstance(item, str) and (sheet_name not in ['Data Import Template', 'Data Export']):
 				value = handle_html(item)
 			else:
 				value = item
 
-			if isinstance(item, string_types) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None):
+			if isinstance(item, str) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None):
 				# Remove illegal characters from the string
 				value = re.sub(ILLEGAL_CHARACTERS_RE, '', value)
 
@@ -80,12 +81,12 @@ def handle_html(data):
 
 	return value
 
+
 def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=None):
 	if file_url:
 		_file = frappe.get_doc("File", {"file_url": file_url})
 		filename = _file.get_full_path()
 	elif fcontent:
-		from io import BytesIO
 		filename = BytesIO(fcontent)
 	elif filepath:
 		filename = filepath
@@ -102,6 +103,7 @@ def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=Non
 		rows.append(tmp_list)
 	return rows
 
+
 def read_xls_file_from_attached_file(content):
 	book = xlrd.open_workbook(file_contents=content)
 	sheets = book.sheets()
@@ -111,6 +113,7 @@ def read_xls_file_from_attached_file(content):
 		rows.append(sheet.row_values(i))
 	return rows
 
+
 def build_xlsx_response(data, filename):
 	xlsx_file = make_xlsx(data, filename)
 	# write out response as a xlsx type

frappe/website/doctype/web_page/web_page.py

@@ -242,11 +242,11 @@ def extract_script_and_style_tags(html):
 	styles = []
 
 	for script in soup.find_all('script'):
-		scripts.append(script.text)
+		scripts.append(script.string)
 		script.extract()
 
 	for style in soup.find_all('style'):
-		styles.append(style.text)
+		styles.append(style.string)
 		style.extract()
 
 	return str(soup), scripts, styles

frappe/website/doctype/website_settings/google_indexing.py

@@ -2,17 +2,18 @@
 # Copyright (c) 2020, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
-import google.oauth2.credentials
 
-from frappe import _
+from urllib.parse import quote
+
+import google.oauth2.credentials
+import requests
+from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
-from frappe.utils import get_request_site_address
-from six.moves.urllib.parse import quote
+
+import frappe
+from frappe import _
 from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.utils import get_request_site_address
 
 SCOPES = "https://www.googleapis.com/auth/indexing"
 
@@ -82,7 +83,12 @@ def get_google_indexing_object():
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_indexing = googleapiclient.discovery.build("indexing", "v3", credentials=credentials)
+	google_indexing = build(
+		serviceName="indexing",
+		version="v3",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	return google_indexing
 

frappe/www/login.py

@@ -95,14 +95,6 @@ def login_via_frappe(code, state):
 def login_via_office365(code, state):
 	login_via_oauth2_id_token("office_365", code, state, decoder=decoder_compat)
 
-@frappe.whitelist(allow_guest=True)
-def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=None, generate_login_token=False):
-	if not ((data and provider and state) or (email_id and key)):
-		frappe.respond_as_web_page(_("Invalid Request"), _("Missing parameters for login"), http_status_code=417)
-		return
-
-	_login_oauth_user(data, provider, state, email_id, key, generate_login_token)
-
 @frappe.whitelist(allow_guest=True)
 def login_via_token(login_token):
 	sid = frappe.cache().get_value("login_token:{0}".format(login_token), expires=True)

package.json

@@ -28,7 +28,7 @@
     "driver.js": "^0.9.8",
     "express": "^4.17.1",
     "fast-deep-equal": "^2.0.1",
-    "frappe-charts": "^2.0.0-rc11",
+    "frappe-charts": "^2.0.0-rc13",
     "frappe-datatable": "^1.15.3",
     "frappe-gantt": "^0.5.0",
     "fuse.js": "^3.4.6",

requirements.txt

@@ -1,79 +1,79 @@
-Babel==2.6.0
-beautifulsoup4==4.8.2
-bleach-whitelist==0.0.10
-bleach==3.3.0
-boto3==1.10.18
-braintree==3.57.1
-chardet==3.0.4
-Click==7.0
-coverage==4.5.4
-croniter==0.3.31
-cryptography==3.3.2
-dropbox==9.1.0
-email-reply-parser==0.5.9
-Faker==2.0.4
+Babel~=2.9.0
+beautifulsoup4~=4.9.3
+bleach-whitelist~=0.0.11
+bleach~=3.3.0
+boto3~=1.17.53
+braintree~=4.8.0
+chardet~=4.0.0
+Click~=7.1.2
+coverage~=5.5
+croniter~=1.0.11
+cryptography~=3.4.7
+dropbox~=11.7.0
+email-reply-parser~=0.5.12
+Faker~=8.1.0
 future==0.18.2
-gitdb2==2.0.6;python_version<'3.4'
-GitPython==2.1.15
-git-url-parse==1.2.2
-google-api-python-client==1.9.3
-google-auth-httplib2==0.0.3
-google-auth-oauthlib==0.4.1
-google-auth==1.18.0
-googlemaps==3.1.1
-gunicorn==19.10.0
-html2text==2016.9.19
-html5lib==1.0.1
-ipython==7.14.0
-jedi==0.17.2  # not directly required. Pinned to fix upstream issue with ipython.
-Jinja2==2.11.3
-ldap3==2.7
-markdown2==2.4.0
+git-url-parse~=1.2.2
+gitdb~=4.0.7
+GitPython~=3.1.14
+google-api-python-client~=2.2.0
+google-auth-httplib2~=0.1.0
+google-auth-oauthlib~=0.4.4
+google-auth~=1.29.0
+googlemaps~=4.4.5
+gunicorn~=20.1.0
+html2text==2020.1.16
+html5lib~=1.1
+ipython~=7.16.1
+jedi==0.17.2  # not directly required. Pinned to fix upstream IPython issue (https://github.com/ipython/ipython/issues/12740)
+Jinja2~=2.11.3
+ldap3~=2.9
+markdown2~=2.4.0
 maxminddb-geolite2==2018.703
-ndg-httpsclient==0.5.1
-num2words==0.5.10
-oauthlib==3.1.0
-openpyxl==2.6.4
-passlib==1.7.3
-pdfkit==0.6.1
-Pillow>=8.0.0
-premailer==3.6.1
-psutil==5.7.2
-psycopg2-binary==2.8.4
-pyasn1==0.4.8
-PyJWT==1.7.1
-PyMySQL==0.9.3
-pyngrok==4.1.6
-pyOpenSSL==19.1.0
-pyotp==2.3.0
-PyPDF2==1.26.0
-pypng==0.0.20
-PyQRCode==1.2.1
-python-dateutil==2.8.1
-pytz==2019.3
-PyYAML==5.4
-rauth==0.7.3
-redis==3.5.3
-requests-oauthlib==1.3.0
-requests==2.23.0
-RestrictedPython==5.0
-rq>=1.1.0
-schedule==0.6.0
-semantic-version==2.8.4
-simple-chalk==0.1.0
-six==1.14.0
-sqlparse==0.2.4
-stripe==2.40.0
-terminaltables==3.1.0
-unittest-xml-reporting==2.5.2
-urllib3==1.25.9
-watchdog==0.8.0
-Werkzeug==0.16.1
-Whoosh==2.7.4
-xlrd==1.2.0
-zxcvbn-python==4.4.24
-pycryptodome==3.9.8
-paytmchecksum==1.7.0
-wrapt==1.10.11
-razorpay==1.2.0
+ndg-httpsclient~=0.5.1
+num2words~=0.5.10
+oauthlib~=3.1.0
+openpyxl~=3.0.7
+passlib~=1.7.4
+paytmchecksum~=1.7.0
+pdfkit~=0.6.1
+Pillow~=8.2.0
+premailer~=3.8.0
+psutil~=5.8.0
+psycopg2-binary~=2.8.6
+pyasn1~=0.4.8
+pycryptodome~=3.10.1
+PyJWT~=1.7.1
+PyMySQL~=1.0.2
+pyngrok~=5.0.5
+pyOpenSSL~=20.0.1
+pyotp~=2.6.0
+PyPDF2~=1.26.0
+pypng~=0.0.20
+PyQRCode~=1.2.1
+python-dateutil~=2.8.1
+pytz==2021.1
+PyYAML~=5.4.1
+rauth~=0.7.3
+razorpay~=1.2.0
+redis~=3.5.3
+requests-oauthlib~=1.3.0
+requests~=2.25.1
+RestrictedPython~=5.1
+rq~=1.8.0
 rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability
+schedule~=1.1.0
+semantic-version~=2.8.5
+simple-chalk~=0.1.0
+six~=1.15.0
+sqlparse~=0.4.1
+stripe~=2.56.0
+terminaltables~=3.1.0
+unittest-xml-reporting~=3.0.4
+urllib3~=1.26.4
+watchdog~=2.0.2
+Werkzeug~=0.16.1
+Whoosh~=2.7.4
+wrapt~=1.12.1
+xlrd~=2.0.1
+zxcvbn-python~=4.4.24

yarn.lock

@@ -2699,10 +2699,10 @@ fragment-cache@^0.2.1:
   dependencies:
     map-cache "^0.2.2"
 
-frappe-charts@^2.0.0-rc11:
-  version "2.0.0-rc11"
-  resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc11.tgz#0724fa0d43593362c075c3805ebbbe1a608fcef7"
-  integrity sha512-DY3tThT1lNGcJlRMOtIhnILtSm5h1iKysWhZAyj7yrGiOnOWbZpYx/NZzXZYwtRrWwMlYiLX2ylV76qo31ONsg==
+frappe-charts@^2.0.0-rc13:
+  version "2.0.0-rc13"
+  resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc13.tgz#fdb251d7ae311c41e38f90a3ae108070ec6b9072"
+  integrity sha512-Bv7IfllIrjRbKWHn5b769dOSenqdBixAr6m5kurf8ZUOJSLOgK4HOXItJ7BA8n9PvviH9/k5DaloisjLM2Bm1w==
 
 frappe-datatable@^1.15.3:
   version "1.15.3"